From 99619b6c204b3044b9f2a5227e0a48f6545e0995 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Thu, 8 Jun 2023 19:19:11 +0800 Subject: [PATCH 001/145] set BGW_TEST_MPI_NPROCS --- easybuild/easyblocks/b/berkeleygw.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/b/berkeleygw.py b/easybuild/easyblocks/b/berkeleygw.py index e9a9b87f8e..bb60220a47 100644 --- a/easybuild/easyblocks/b/berkeleygw.py +++ b/easybuild/easyblocks/b/berkeleygw.py @@ -177,7 +177,8 @@ def test_step(self): """Custom test step for BerkeleyGW.""" if self.cfg['runtest'] is not False: self.cfg['runtest'] = 'check' - setvar('OMP_NUM_THREADS', '4') + setvar('BGW_TEST_MPI_NPROCS', '2') + setvar('OMP_NUM_THREADS', '2') setvar('TEMPDIRPATH', os.path.join(self.builddir, 'tmp')) super(EB_BerkeleyGW, self).test_step() From 9483d0a4af174712dc879ba39e1ed58f73fa0d2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Tue, 18 Jul 2023 19:18:07 +0200 Subject: [PATCH 002/145] adding easyblocks: cargopythonbundle.py --- .../easyblocks/generic/cargopythonbundle.py | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 easybuild/easyblocks/generic/cargopythonbundle.py diff --git a/easybuild/easyblocks/generic/cargopythonbundle.py b/easybuild/easyblocks/generic/cargopythonbundle.py new file mode 100644 index 0000000000..f1a4ac97fd --- /dev/null +++ b/easybuild/easyblocks/generic/cargopythonbundle.py @@ -0,0 +1,59 @@ +## +# Copyright 2018-2023 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for installing python bundles with some packages using Rust + +@author: Mikael Oehman (Chalmers University of Technology) +""" + +from easybuild.easyblocks.generic.cargo import Cargo +from easybuild.easyblocks.generic.pythonbundle import PythonBundle + + +class CargoPythonBundle(PythonBundle, Cargo): # PythonBundle must come first to take precedence + """ + Builds just like PythonBundle with setup for Rust and crates from Cargo easyblock + + The cargo init step will set up the environment variables for rustc and vendor sources + but all the build steps are triggered like normal. + """ + + @staticmethod + def extra_options(extra_vars=None): + """Define extra easyconfig parameters specific to Cargo""" + extra_vars = PythonBundle.extra_options(extra_vars) + extra_vars = Cargo.extra_options(extra_vars) # not all extra options here will used here + + return extra_vars + + def __init__(self, *args, **kwargs): + """Constructor for CargoPythonBundle easyblock.""" + Cargo.__init__(self, *args, **kwargs) + PythonBundle.__init__(self, *args, **kwargs) + + def extract_step(self): + """Specifically use the overloaded variant from Cargo as is populates vendored sources with checksums.""" + return Cargo.extract_step(self) + From 8104581bd1b7a729b265e900ab4b41d88acdfdfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Tue, 18 Jul 2023 19:18:49 +0200 Subject: [PATCH 003/145] Fix style --- easybuild/easyblocks/generic/cargopythonbundle.py | 1 - 1 file changed, 1 deletion(-) diff --git a/easybuild/easyblocks/generic/cargopythonbundle.py b/easybuild/easyblocks/generic/cargopythonbundle.py index f1a4ac97fd..c4941dad0f 100644 --- a/easybuild/easyblocks/generic/cargopythonbundle.py +++ b/easybuild/easyblocks/generic/cargopythonbundle.py @@ -56,4 +56,3 @@ def __init__(self, *args, **kwargs): def extract_step(self): """Specifically use the overloaded variant from Cargo as is populates vendored sources with checksums.""" return Cargo.extract_step(self) - From ec46c13578893ee712c9b1e7216ad7602d3352b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Wed, 19 Jul 2023 11:30:30 +0200 Subject: [PATCH 004/145] Opt out of source list check in bundle --- easybuild/easyblocks/generic/bundle.py | 11 ++++++----- easybuild/easyblocks/generic/cargopythonbundle.py | 3 +-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/easybuild/easyblocks/generic/bundle.py b/easybuild/easyblocks/generic/bundle.py index 61cc7125fd..4b0f798323 100644 --- a/easybuild/easyblocks/generic/bundle.py +++ b/easybuild/easyblocks/generic/bundle.py @@ -65,7 +65,7 @@ def extra_options(extra_vars=None): }) return EasyBlock.extra_options(extra_vars) - def __init__(self, *args, **kwargs): + def __init__(self, *args, check_for_sources=True, **kwargs): """Initialize easyblock.""" super(Bundle, self).__init__(*args, **kwargs) self.altroot = None @@ -78,10 +78,11 @@ def __init__(self, *args, **kwargs): self.comp_cfgs_sanity_check = [] # list of sources for bundle itself *must* be empty - if self.cfg['sources']: - raise EasyBuildError("List of sources for bundle itself must be empty, found %s", self.cfg['sources']) - if self.cfg['patches']: - raise EasyBuildError("List of patches for bundle itself must be empty, found %s", self.cfg['patches']) + if check_for_sources: + if self.cfg['sources']: + raise EasyBuildError("List of sources for bundle itself must be empty, found %s", self.cfg['sources']) + if self.cfg['patches']: + raise EasyBuildError("List of patches for bundle itself must be empty, found %s", self.cfg['patches']) # disable templating to avoid premature resolving of template values self.cfg.enable_templating = False diff --git a/easybuild/easyblocks/generic/cargopythonbundle.py b/easybuild/easyblocks/generic/cargopythonbundle.py index c4941dad0f..2cbec2dd94 100644 --- a/easybuild/easyblocks/generic/cargopythonbundle.py +++ b/easybuild/easyblocks/generic/cargopythonbundle.py @@ -50,8 +50,7 @@ def extra_options(extra_vars=None): def __init__(self, *args, **kwargs): """Constructor for CargoPythonBundle easyblock.""" - Cargo.__init__(self, *args, **kwargs) - PythonBundle.__init__(self, *args, **kwargs) + super(CargoPythonBundle, self).__init__(*args, check_for_sources=False, **kwargs) def extract_step(self): """Specifically use the overloaded variant from Cargo as is populates vendored sources with checksums.""" From 364aeae337e299cb462f99935150f43eaaff6d8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Wed, 19 Jul 2023 09:45:20 +0000 Subject: [PATCH 005/145] Set EBROOTPYTHON to CargoPythonBundle test --- test/easyblocks/module.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/easyblocks/module.py b/test/easyblocks/module.py index db44834031..1fac194147 100644 --- a/test/easyblocks/module.py +++ b/test/easyblocks/module.py @@ -46,6 +46,7 @@ from easybuild.easyblocks.generic.juliapackage import JuliaPackage from easybuild.easyblocks.generic.intelbase import IntelBase from easybuild.easyblocks.generic.pythonbundle import PythonBundle +from easybuild.easyblocks.generic.cargopythonbundle import CargoPythonBundle from easybuild.easyblocks.gcc import EB_GCC from easybuild.easyblocks.imod import EB_IMOD from easybuild.easyblocks.fftwmpi import EB_FFTW_period_MPI @@ -279,7 +280,7 @@ def template_module_only_test(self, easyblock, name, version='1.3.2', extra_txt= # $JAVA_HOME must be set for IMOD os.environ['JAVA_HOME'] = tmpdir - elif app_class == PythonBundle: + elif app_class == PythonBundle or app_class == CargoPythonBundle: # $EBROOTPYTHON must be set for PythonBundle easyblock os.environ['EBROOTPYTHON'] = '/fake/install/prefix/Python/2.7.14-foss-2018a' From 1db04efdfeb58ed6ada5301e7ee5cf1543eee3f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Wed, 19 Jul 2023 23:17:11 +0200 Subject: [PATCH 006/145] Reorder args for bundle to work in python 2 --- easybuild/easyblocks/generic/bundle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/bundle.py b/easybuild/easyblocks/generic/bundle.py index 4b0f798323..a1b8593c7e 100644 --- a/easybuild/easyblocks/generic/bundle.py +++ b/easybuild/easyblocks/generic/bundle.py @@ -65,7 +65,7 @@ def extra_options(extra_vars=None): }) return EasyBlock.extra_options(extra_vars) - def __init__(self, *args, check_for_sources=True, **kwargs): + def __init__(self, check_for_sources=True, *args, **kwargs): """Initialize easyblock.""" super(Bundle, self).__init__(*args, **kwargs) self.altroot = None From b4c02702fb8a273bb970abfebe7573f569db96b3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 5 Sep 2023 19:49:02 +0200 Subject: [PATCH 007/145] add custom named argument 'check_for_sources' for Bundle constructor in Python 2+3 compatible way --- easybuild/easyblocks/generic/bundle.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/bundle.py b/easybuild/easyblocks/generic/bundle.py index a1b8593c7e..8ae2432913 100644 --- a/easybuild/easyblocks/generic/bundle.py +++ b/easybuild/easyblocks/generic/bundle.py @@ -65,7 +65,12 @@ def extra_options(extra_vars=None): }) return EasyBlock.extra_options(extra_vars) - def __init__(self, check_for_sources=True, *args, **kwargs): + def __init__(self, *args, **kwargs): + """ + Constructor for Bundle easyblock + """ + # add check_for_sources named argument in a Python 2+3 compatible way + check_for_sources = kwargs.get('check_for_sources', True) """Initialize easyblock.""" super(Bundle, self).__init__(*args, **kwargs) self.altroot = None From 04a99c3201abb8a6fcceb228d8a1008d465fad57 Mon Sep 17 00:00:00 2001 From: Stefan Wolfsheimer Date: Thu, 7 Sep 2023 22:45:08 +0200 Subject: [PATCH 008/145] wrf: extract version number --- easybuild/easyblocks/w/wrf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/w/wrf.py b/easybuild/easyblocks/w/wrf.py index b232400724..6c8dd16ea3 100644 --- a/easybuild/easyblocks/w/wrf.py +++ b/easybuild/easyblocks/w/wrf.py @@ -54,7 +54,7 @@ def det_wrf_subdir(wrf_version): """Determine WRF subdirectory for given WRF version.""" - if LooseVersion(wrf_version) < LooseVersion('4.0'): + if LooseVersion(wrf_version) < LooseVersion('4.0') or wrf_version == '4.5.1': wrf_subdir = 'WRFV%s' % wrf_version.split('.')[0] else: wrf_subdir = 'WRF-%s' % wrf_version From 13ff69f5e22ca53d67cd1e207abc097a1df14d5e Mon Sep 17 00:00:00 2001 From: Stefan Wolfsheimer Date: Fri, 8 Sep 2023 09:53:37 +0200 Subject: [PATCH 009/145] determine wrf_subdir for version >= 4.5.1 --- easybuild/easyblocks/w/wrf.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/w/wrf.py b/easybuild/easyblocks/w/wrf.py index 6c8dd16ea3..ddd6387fd8 100644 --- a/easybuild/easyblocks/w/wrf.py +++ b/easybuild/easyblocks/w/wrf.py @@ -54,8 +54,10 @@ def det_wrf_subdir(wrf_version): """Determine WRF subdirectory for given WRF version.""" - if LooseVersion(wrf_version) < LooseVersion('4.0') or wrf_version == '4.5.1': + if LooseVersion(wrf_version) < LooseVersion('4.0'): wrf_subdir = 'WRFV%s' % wrf_version.split('.')[0] + elif LooseVersion(wrf_version) >= LooseVersion('4.5.1'): + wrf_subdir = 'WRFV%s' % wrf_version else: wrf_subdir = 'WRF-%s' % wrf_version From 1195cd76e0b80bc5e8b31e61f7419e6730f6b7e5 Mon Sep 17 00:00:00 2001 From: c3-builder Date: Mon, 18 Sep 2023 12:57:32 +0200 Subject: [PATCH 010/145] Install only suitespare c libraries with --- easybuild/easyblocks/s/suitesparse.py | 238 ++++++++++++-------------- 1 file changed, 105 insertions(+), 133 deletions(-) diff --git a/easybuild/easyblocks/s/suitesparse.py b/easybuild/easyblocks/s/suitesparse.py index f25fc66785..430d99ea99 100644 --- a/easybuild/easyblocks/s/suitesparse.py +++ b/easybuild/easyblocks/s/suitesparse.py @@ -61,24 +61,22 @@ def configure_step(self): if LooseVersion(self.version) < LooseVersion('4.0'): self.config_name = 'UFconfig' - else: + elif LooseVersion(self.version) < LooseVersion('6.0.0'): self.config_name = 'SuiteSparse_config' + else: + # config file is removed after v6.0.0 + self.config_name = '' cfgvars = { - 'CC': os.getenv('MPICC'), + 'CC': os.getenv('CC'), 'CFLAGS': os.getenv('CFLAGS'), - 'CXX': os.getenv('MPICXX'), - 'F77': os.getenv('MPIF77'), + 'CXX': os.getenv('CXX'), + 'F77': os.getenv('F77'), 'F77FLAGS': os.getenv('F77FLAGS'), + 'BLAS': os.getenv('LIBBLAS_MT'), + 'LAPACK': os.getenv('LIBLAPACK_MT'), } - # avoid that (system) Intel compilers are always considered - self.cfg.update('buildopts', 'AUTOCC=no') - - # Set BLAS and LAPACK libraries as specified in SuiteSparse README.txt - self.cfg.update('buildopts', 'BLAS="%s"' % os.getenv('LIBBLAS_MT')) - self.cfg.update('buildopts', 'LAPACK="%s"' % os.getenv('LIBLAPACK_MT')) - # Get CUDA and set it up appropriately cuda = get_software_root('CUDA') if cuda: @@ -91,125 +89,106 @@ def configure_step(self): # Get METIS or ParMETIS settings metis = get_software_root('METIS') parmetis = get_software_root('ParMETIS') - if parmetis: - metis_path = parmetis - metis_include = os.path.join(parmetis, 'include') - metis_libs = os.path.join(parmetis, get_software_libdir('ParMETIS'), 'libmetis.a') - - elif metis: - metis_path = metis - metis_include = os.path.join(metis, 'include') - metis_libs = os.path.join(metis, get_software_libdir('METIS'), 'libmetis.a') + if parmetis or metis: + if parmetis: + metis_name = 'ParMETIS' + elif metis: + metis_name = 'METIS' + metis_path = get_software_root(metis_name) + metis_include = os.path.join(metis_path, 'include') + metis_libs = os.path.join(metis_path, get_software_libdir(metis_name), 'libmetis.a') else: - raise EasyBuildError("Neither METIS or ParMETIS module loaded.") - - if LooseVersion(self.version) >= LooseVersion('4.5.1'): + self.log.info("Use METIS built in SuiteSparse") + # raise EasyBuildError("Neither METIS or ParMETIS module loaded.") + + # config file can catch environment variables after v4.5.0 + if LooseVersion(self.version) < LooseVersion('4.5.0'): cfgvars.update({ - 'MY_METIS_LIB': metis_libs, - 'MY_METIS_INC': metis_include, - }) - else: - cfgvars.update({ - 'METIS_PATH': metis_path, - 'METIS': metis_libs, + 'INSTALL_LIB': os.path.join(self.installdir, 'lib'), + 'INSTALL_INCLUDE': os.path.join(self.installdir, 'include'), }) + if parmetis or metis: + cfgvars.update({ + 'METIS_PATH': metis_path, + 'METIS': metis_libs, + }) + + # patch file + fp = os.path.join(self.cfg['start_dir'], self.config_name, '%s.mk' % self.config_name) + + try: + for line in fileinput.input(fp, inplace=1, backup='.orig'): + for (var, val) in list(cfgvars.items()): + # Let's overwrite NVCCFLAGS at the end, since the line breaks and the fact that it appears multiple + # times makes it tricky to handle it properly + # path variables are also moved to the end + if var not in ['NVCCFLAGS', 'INSTALL_LIB', 'INSTALL_INCLUDE', 'METIS_PATH']: + orig_line = line + # for variables in cfgvars, substiture lines assignment + # in the file, whatever they are, by assignments to the + # values in cfgvars + line = re.sub(r"^\s*(%s\s*=\s*).*\n$" % var, + r"\1 %s # patched by EasyBuild\n" % val, + line) + if line != orig_line: + cfgvars.pop(var) + sys.stdout.write(line) + except IOError as err: + raise EasyBuildError("Failed to patch %s in: %s", fp, err) + + # add remaining entries at the end + if cfgvars: + cfgtxt = '# lines below added automatically by EasyBuild\n' + cfgtxt += '\n'.join(["%s = %s" % (var, val) for (var, val) in cfgvars.items()]) + write_file(fp, cfgtxt, append=True) + + elif LooseVersion(self.version) < LooseVersion('6.0.0'): + # avoid that (system) Intel compilers are always considered + self.cfg.update('prebuildopts', 'AUTOCC=no') + + # Set BLAS and LAPACK libraries as specified in SuiteSparse README.txt + self.cfg.update('buildopts', 'BLAS="%s"' % cfgvars.get('BLAS')) + self.cfg.update('buildopts', 'LAPACK="%s"' % cfgvars.get('LAPACK')) + + self.cfg.update('installopts', 'INSTALL="%s"' % self.installdir) + self.cfg.update('installopts', 'BLAS="%s"' % cfgvars.get('BLAS')) + self.cfg.update('installopts', 'LAPACK="%s"' % cfgvars.get('LAPACK')) + + if LooseVersion(self.version) >= LooseVersion('5.1.2'): + # graphblas exists, needs cmake + # v5.0.0 until v5.1.2 has no CMAKE_OPTIONS to set + # probably need patch + self.cfg.update('preinstallopts', 'CMAKE_OPTIONS="-DCMAKE_INSTALL_PREFIX=%s"' % self.installdir) + + # set METIS library + if parmetis or metis: + if LooseVersion(self.version) == LooseVersion('4.5.0'): + self.cfg.update('buildopts', 'METIS_PATH="%s"' % metis_path) + self.cfg.update('installopts', 'METIS_PATH="%s"' % metis_path) + else: + self.cfg.update('buildopts', 'MY_METIS_LIB="%s"' % metis_libs) + self.cfg.update('buildopts', 'MY_METIS_INC="%s"' % metis_include) + self.cfg.update('installopts', 'MY_METIS_LIB="%s"' % metis_libs) + self.cfg.update('installopts', 'MY_METIS_INC="%s"' % metis_include) - # patch file - fp = os.path.join(self.cfg['start_dir'], self.config_name, '%s.mk' % self.config_name) - - try: - for line in fileinput.input(fp, inplace=1, backup='.orig'): - for (var, val) in list(cfgvars.items()): - # Let's overwrite NVCCFLAGS at the end, since the line breaks and the fact that it appears multiple - # times makes it tricky to handle it properly - if var != 'NVCCFLAGS': - orig_line = line - # for variables in cfgvars, substiture lines assignment - # in the file, whatever they are, by assignments to the - # values in cfgvars - line = re.sub(r"^\s*(%s\s*=\s*).*\n$" % var, - r"\1 %s # patched by EasyBuild\n" % val, - line) - if line != orig_line: - cfgvars.pop(var) - sys.stdout.write(line) - except IOError as err: - raise EasyBuildError("Failed to patch %s in: %s", fp, err) - - # add remaining entries at the end - if cfgvars: - cfgtxt = '# lines below added automatically by EasyBuild\n' - cfgtxt += '\n'.join(["%s = %s" % (var, val) for (var, val) in cfgvars.items()]) - write_file(fp, cfgtxt, append=True) + else: + # after v6.0.0, no option for metis, its own metis is used anyway + # nothing to do here, set the CMAKE_OPTIONS in easyconfigs + pass + # self.cfg.update( + # 'prebuildopts', + # 'CMAKE_OPTIONS="-DCMAKE_INSTALL_PREFIX=%s -DBLA_VENDOR=FlexiBLAS"' % self.installdir + # ) def install_step(self): """Install by copying the contents of the builddir to the installdir (preserving permissions)""" - for x in os.listdir(self.cfg['start_dir']): - src = os.path.join(self.cfg['start_dir'], x) - dst = os.path.join(self.installdir, x) - try: - if os.path.isdir(src): - # symlink points to CUDA folder that is - # not created for non GPU nodes. shutil - # throws an error in this case. - copy_dir(src, dst, symlinks=True) - # symlink - # - dst/Lib to dst/lib - # - dst/Include to dst/include - for c in ['Lib', 'Include']: - nsrc = os.path.join(dst, c) - ndst = os.path.join(dst, c.lower()) - if os.path.exists(nsrc): - os.symlink(nsrc, ndst) - # enable r-x permissions for group/others - perms = stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH - adjust_permissions(dst, perms, add=True, recursive=True, onlydirs=True) - else: - shutil.copy2(src, dst) - except OSError as err: - raise EasyBuildError("Copying src %s to dst %s failed: %s", src, dst, err) - - # some extra symlinks are necessary for UMFPACK to work. - paths = [ - os.path.join('AMD', 'include', 'amd.h'), - os.path.join('AMD', 'include', 'amd_internal.h'), - os.path.join(self.config_name, '%s.h' % self.config_name), - os.path.join('AMD', 'lib', 'libamd.a') - ] - for path in paths: - src = os.path.join(self.installdir, path) - dn = path.split(os.path.sep)[-2] - fn = path.split(os.path.sep)[-1] - dstdir = os.path.join(self.installdir, 'UMFPACK', dn) - mkdir(dstdir) - if os.path.exists(src): - try: - os.symlink(src, os.path.join(dstdir, fn)) - except OSError as err: - raise EasyBuildError("Failed to make symbolic link from %s to %s: %s", src, dst, err) - - def make_module_req_guess(self): - """ - Extra path to consider for module file: - * add config dir and include to $CPATH so include files are found - * add UMFPACK and AMD library, and lib dirs to $LD_LIBRARY_PATH - """ - - guesses = super(EB_SuiteSparse, self).make_module_req_guess() - - # Previous versions of SuiteSparse used specific directories for includes and libraries - if LooseVersion(self.version) < LooseVersion('4.5'): - include_dirs = [self.config_name] - ld_library_path = ['AMD/lib', 'BTF/lib', 'CAMD/lib', 'CCOLAMD/lib', 'CHOLAMD/lib', 'CHOLMOD/lib', - 'COLAMD/lib/', 'CSparse/lib', 'CXSparse/lib', 'KLU/lib', 'LDL/lib', 'RBio/lib', - 'UMFPACK/lib', self.config_name] - - guesses['CPATH'].extend(include_dirs) - guesses['LD_LIBRARY_PATH'].extend(ld_library_path) - guesses['LIBRARY_PATH'].extend(ld_library_path) - return guesses + if LooseVersion(self.version) < LooseVersion('4.5.0'): + mkdir(os.path.join(self.installdir, 'lib')) + mkdir(os.path.join(self.installdir, 'include')) + + super(EB_SuiteSparse, self).install_step() def sanity_check_step(self): """Custom sanity check for SuiteSparse.""" @@ -219,24 +198,17 @@ def sanity_check_step(self): raise EasyBuildError("SuiteSparse has compiled its own Metis. This will conflict with the Metis build." " The SuiteSparse EasyBlock need to be updated!") + shlib_ext = get_shared_lib_ext() libnames = ['AMD', 'BTF', 'CAMD', 'CCOLAMD', 'CHOLMOD', 'COLAMD', 'CXSparse', 'KLU', 'LDL', 'RBio', 'SPQR', 'UMFPACK'] - libs = [os.path.join(x, 'lib', 'lib%s.a' % x.lower()) for x in libnames] - - if LooseVersion(self.version) < LooseVersion('4.0'): - csparse_dir = 'CSparse3' + if LooseVersion(self.version) < LooseVersion('4.5'): + libs = [os.path.join('lib', 'lib%s.a' % x.lower()) for x in libnames] else: - csparse_dir = 'CSparse' - libs.append(os.path.join(csparse_dir, 'lib', 'libcsparse.a')) - - # Latest version of SuiteSparse also compiles shared library and put them in 'lib' - shlib_ext = get_shared_lib_ext() - if LooseVersion(self.version) >= LooseVersion('4.5.1'): - libs += [os.path.join('lib', 'lib%s.%s' % (x.lower(), shlib_ext)) for x in libnames] + libs = [os.path.join('lib', 'lib%s.%s' % (x.lower(), shlib_ext)) for x in libnames] custom_paths = { 'files': libs, - 'dirs': ['MATLAB_Tools'], + 'dirs': [], } super(EB_SuiteSparse, self).sanity_check_step(custom_paths=custom_paths) From bfdefd932ffb694815fcd422af9dda39828ca9b8 Mon Sep 17 00:00:00 2001 From: c3-builder Date: Mon, 18 Sep 2023 14:58:21 +0200 Subject: [PATCH 011/145] Clean up the codes --- easybuild/easyblocks/s/suitesparse.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/easybuild/easyblocks/s/suitesparse.py b/easybuild/easyblocks/s/suitesparse.py index 430d99ea99..75812024f9 100644 --- a/easybuild/easyblocks/s/suitesparse.py +++ b/easybuild/easyblocks/s/suitesparse.py @@ -101,7 +101,7 @@ def configure_step(self): else: self.log.info("Use METIS built in SuiteSparse") # raise EasyBuildError("Neither METIS or ParMETIS module loaded.") - + # config file can catch environment variables after v4.5.0 if LooseVersion(self.version) < LooseVersion('4.5.0'): cfgvars.update({ @@ -116,12 +116,12 @@ def configure_step(self): # patch file fp = os.path.join(self.cfg['start_dir'], self.config_name, '%s.mk' % self.config_name) - + try: for line in fileinput.input(fp, inplace=1, backup='.orig'): for (var, val) in list(cfgvars.items()): - # Let's overwrite NVCCFLAGS at the end, since the line breaks and the fact that it appears multiple - # times makes it tricky to handle it properly + # Let's overwrite NVCCFLAGS at the end, since the line breaks and + # the fact that it appears multiple times makes it tricky to handle it properly # path variables are also moved to the end if var not in ['NVCCFLAGS', 'INSTALL_LIB', 'INSTALL_INCLUDE', 'METIS_PATH']: orig_line = line @@ -176,10 +176,6 @@ def configure_step(self): # after v6.0.0, no option for metis, its own metis is used anyway # nothing to do here, set the CMAKE_OPTIONS in easyconfigs pass - # self.cfg.update( - # 'prebuildopts', - # 'CMAKE_OPTIONS="-DCMAKE_INSTALL_PREFIX=%s -DBLA_VENDOR=FlexiBLAS"' % self.installdir - # ) def install_step(self): """Install by copying the contents of the builddir to the installdir (preserving permissions)""" @@ -187,7 +183,7 @@ def install_step(self): if LooseVersion(self.version) < LooseVersion('4.5.0'): mkdir(os.path.join(self.installdir, 'lib')) mkdir(os.path.join(self.installdir, 'include')) - + super(EB_SuiteSparse, self).install_step() def sanity_check_step(self): From d5615c9b590acf5084c69854e1444e8555d01106 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 19 Sep 2023 13:12:35 +0200 Subject: [PATCH 012/145] remove stray newline in pytorch error message --- easybuild/easyblocks/p/pytorch.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/easybuild/easyblocks/p/pytorch.py b/easybuild/easyblocks/p/pytorch.py index 40c116b197..ca98844428 100644 --- a/easybuild/easyblocks/p/pytorch.py +++ b/easybuild/easyblocks/p/pytorch.py @@ -318,7 +318,7 @@ def get_count_for_pattern(regex, text): return 0 # Create clear summary report - failure_report = "" + failure_report = [] failure_cnt = 0 error_cnt = 0 failed_test_suites = [] @@ -337,9 +337,9 @@ def get_count_for_pattern(regex, text): # E.g. 'failures=3, errors=10, skipped=190, expected failures=6' failure_summary = m.group('failure_summary') total, test_suite = m.group('test_cnt', 'failed_test_suite_name') - failure_report += "{test_suite} ({total} total tests, {failure_summary})\n".format( + failure_report.append("{test_suite} ({total} total tests, {failure_summary})".format( test_suite=test_suite, total=total, failure_summary=failure_summary - ) + )) failure_cnt += get_count_for_pattern(r"(? Date: Mon, 25 Sep 2023 11:11:41 +0200 Subject: [PATCH 013/145] fix sanity-check-only and module-only for UCX plugins --- easybuild/easyblocks/u/ucx_plugins.py | 36 ++++++++++++++++++--------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/easybuild/easyblocks/u/ucx_plugins.py b/easybuild/easyblocks/u/ucx_plugins.py index 10e5a97e13..0987487ed1 100644 --- a/easybuild/easyblocks/u/ucx_plugins.py +++ b/easybuild/easyblocks/u/ucx_plugins.py @@ -43,16 +43,36 @@ class EB_UCX_Plugins(ConfigureMake): """Support for building additional plugins for a existing UCX module""" def __init__(self, *args, **kwargs): - """Custom initialization for UCX: set correct module name.""" + """Custom initialization for UCX-Plugins.""" super(EB_UCX_Plugins, self).__init__(*args, **kwargs) - self.plugins = {} + self._plugins = None self.makefile_dirs = [] + @property + def plugins(self): + if self._plugins is None: + plugins = defaultdict(list) + + if get_software_root('CUDAcore') or get_software_root('CUDA'): + for key in ('ucm', 'uct', 'ucx_perftest'): + plugins[key].append('cuda') + + if get_software_root('GDRCopy'): + plugins['uct_cuda'].append('gdrcopy') + + # To be supported in the future: + if get_software_root('ROCm'): + for key in ('ucm', 'uct', 'ucx_perftest'): + plugins[key].append('rocm') + + self._plugins = dict(plugins) + print("plugins", plugins) + return self._plugins + def configure_step(self): """Customize configuration for building requested plugins.""" # make sure that required dependencies are loaded - ucxroot = get_software_root('UCX') - if not ucxroot: + if not get_software_root('UCX'): raise EasyBuildError("UCX is a required dependency") self.cfg['configure_cmd'] = 'contrib/configure-release' @@ -62,16 +82,12 @@ def configure_step(self): # omit the lib subdirectory since we are just installing plugins configopts += '--libdir=%(installdir)s ' - plugins = defaultdict(list) cudaroot = get_software_root('CUDAcore') or get_software_root('CUDA') if cudaroot: configopts += '--with-cuda=%s ' % cudaroot - for key in ('ucm', 'uct', 'ucx_perftest'): - plugins[key].append('cuda') gdrcopyroot = get_software_root('GDRCopy') if gdrcopyroot: - plugins['uct_cuda'].append('gdrcopy') configopts += '--with-gdrcopy=%s ' % gdrcopyroot self.makefile_dirs.extend(os.path.join(x, 'cuda') for x in ('uct', 'ucm', 'tools/perf')) @@ -79,13 +95,9 @@ def configure_step(self): # To be supported in the future: rocmroot = get_software_root('ROCm') if rocmroot: - for key in ('ucm', 'uct', 'ucx_perftest'): - plugins[key].append('rocm') configopts += '--with-rocm=%s ' % rocmroot self.makefile_dirs.extend(os.path.join(x, 'rocm') for x in ('uct', 'ucm', 'tools/perf')) - self.plugins = dict(plugins) - self.cfg.update('configopts', configopts) super(EB_UCX_Plugins, self).configure_step() From 7e00cc1f25a31eac78a0de6389d0424ededb2601 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 25 Sep 2023 11:29:14 +0200 Subject: [PATCH 014/145] Replace debug print --- easybuild/easyblocks/u/ucx_plugins.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/u/ucx_plugins.py b/easybuild/easyblocks/u/ucx_plugins.py index 0987487ed1..1523ed8ba4 100644 --- a/easybuild/easyblocks/u/ucx_plugins.py +++ b/easybuild/easyblocks/u/ucx_plugins.py @@ -29,6 +29,7 @@ @author: Mikael Öhman (Chalmers University of Techonology) """ from collections import defaultdict +from itertools import chain import os from easybuild.easyblocks.generic.configuremake import ConfigureMake @@ -66,7 +67,7 @@ def plugins(self): plugins[key].append('rocm') self._plugins = dict(plugins) - print("plugins", plugins) + self.log.info("Creating plugins for %s", ", ".join(sorted(set(chain(*plugins.values()))))) return self._plugins def configure_step(self): From 8f64d316aaa49f616a9980be432c0a84b5d24886 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 27 Sep 2023 16:17:21 +0200 Subject: [PATCH 015/145] add extra newline above statement that collapsed failure_report from list of strings to a single string value in PyTorch easyblock --- easybuild/easyblocks/p/pytorch.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/easyblocks/p/pytorch.py b/easybuild/easyblocks/p/pytorch.py index ca98844428..58a5e9bdb8 100644 --- a/easybuild/easyblocks/p/pytorch.py +++ b/easybuild/easyblocks/p/pytorch.py @@ -409,6 +409,7 @@ def get_count_for_pattern(regex, text): failure_report.extend('+ %s' % t for t in sorted(all_failed_test_suites - failed_test_suites)) # Test suites not included in the catch-all regexp but counted. Should be empty. failure_report.extend('? %s' % t for t in sorted(failed_test_suites - all_failed_test_suites)) + failure_report = '\n'.join(failure_report) # Calculate total number of unsuccesful and total tests From c50753b92db2b51ebf910159a2d48ae2e21c2623 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 27 Sep 2023 16:26:35 +0200 Subject: [PATCH 016/145] add docstring for `plugins` property in UCX plugins easyblock --- easybuild/easyblocks/u/ucx_plugins.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/easyblocks/u/ucx_plugins.py b/easybuild/easyblocks/u/ucx_plugins.py index 1523ed8ba4..2ece31b019 100644 --- a/easybuild/easyblocks/u/ucx_plugins.py +++ b/easybuild/easyblocks/u/ucx_plugins.py @@ -51,6 +51,7 @@ def __init__(self, *args, **kwargs): @property def plugins(self): + """Property to determine list of plugins based on loaded dependencies, or return cached list of plugins.""" if self._plugins is None: plugins = defaultdict(list) From b0012bb9973bf76eff1c601ecd1b0847e491f46a Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Fri, 29 Sep 2023 15:24:23 +0200 Subject: [PATCH 017/145] add new easyblock for HPCC --- easybuild/easyblocks/h/hpcc.py | 84 ++++++++++++++++++++++++++++++++++ easybuild/easyblocks/h/hpl.py | 16 ++++--- 2 files changed, 94 insertions(+), 6 deletions(-) create mode 100644 easybuild/easyblocks/h/hpcc.py diff --git a/easybuild/easyblocks/h/hpcc.py b/easybuild/easyblocks/h/hpcc.py new file mode 100644 index 0000000000..302305f129 --- /dev/null +++ b/easybuild/easyblocks/h/hpcc.py @@ -0,0 +1,84 @@ +## +# Copyright 2009-2023 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for building and installing HPCC, implemented as an easyblock + +@author: Samuel Moors (Vrije Universiteit Brussel) +""" + +import os +import shutil + +from easybuild.easyblocks.h.hpl import EB_HPL +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.filetools import change_dir, remove_file, symlink +from easybuild.tools.run import run_cmd + + +class EB_HPCC(EB_HPL): + """ + Support for building HPCC (HPC Challenge) + - create Make.UNKNOWN + - build with make and install + """ + + def configure_step(self): + """ + Create Make.UNKNOWN file to build from + - provide subdir argument so this can be reused in HPCC easyblock + """ + super(EB_HPCC, self).configure_step(subdir='hpl') + + def build_step(self): + """ + Build with make and correct make options + """ + super(EB_HPCC, self).build_step(topdir='../../..') + + def install_step(self): + """ + Install by copying files to install dir + """ + srcdir = self.cfg['start_dir'] + destdir = os.path.join(self.installdir, 'bin') + filename = 'hpcc' + try: + os.makedirs(destdir) + srcfile = os.path.join(srcdir, filename) + shutil.copy2(srcfile, destdir) + except OSError as err: + raise EasyBuildError("Copying %s to installation dir %s failed: %s", srcfile, destdir, err) + + def sanity_check_step(self): + """ + Custom sanity check for HPL + """ + + custom_paths = { + 'files': ["bin/hpcc"], + 'dirs': [] + } + + super(EB_HPL, self).sanity_check_step(custom_paths) diff --git a/easybuild/easyblocks/h/hpl.py b/easybuild/easyblocks/h/hpl.py index d408a7b05e..b4066969dd 100644 --- a/easybuild/easyblocks/h/hpl.py +++ b/easybuild/easyblocks/h/hpl.py @@ -37,13 +37,14 @@ from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.filetools import change_dir, remove_file, symlink from easybuild.tools.run import run_cmd class EB_HPL(ConfigureMake): """ Support for building HPL (High Performance Linpack) - - create Make.UNKNOWN + - creat Make.UNKNOWN - build with make and install """ @@ -62,7 +63,7 @@ def configure_step(self, subdir=None): setupdir = os.path.join(basedir, 'setup') try: - os.chdir(setupdir) + change_dir(setupdir) except OSError as err: raise EasyBuildError("Failed to change to to dir %s: %s", setupdir, err) @@ -71,14 +72,15 @@ def configure_step(self, subdir=None): run_cmd(cmd, log_all=True, simple=True, log_output=True) try: - os.symlink(os.path.join(setupdir, 'Make.UNKNOWN'), os.path.join(makeincfile)) + remove_file(os.path.join(makeincfile)) + symlink(os.path.join(setupdir, 'Make.UNKNOWN'), os.path.join(makeincfile)) except OSError as err: raise EasyBuildError("Failed to symlink Make.UNKNOWN from %s to %s: %s", setupdir, makeincfile, err) # go back - os.chdir(self.cfg['start_dir']) + change_dir(self.cfg['start_dir']) - def build_step(self): + def build_step(self, topdir=None): """ Build with make and correct make options """ @@ -89,7 +91,9 @@ def build_step(self): raise EasyBuildError("Required environment variable %s not found (no toolchain used?).", envvar) # build dir - extra_makeopts = 'TOPdir="%s" ' % self.cfg['start_dir'] + if not topdir: + topdir = self.cfg['start_dir'] + extra_makeopts = 'TOPdir="%s" ' % topdir # compilers extra_makeopts += 'CC="%(mpicc)s" MPICC="%(mpicc)s" LINKER="%(mpicc)s" ' % {'mpicc': os.getenv('MPICC')} From 38e9f1093966ada1388396f018b57504fa9849b2 Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Fri, 29 Sep 2023 15:26:03 +0200 Subject: [PATCH 018/145] remove unused imports --- easybuild/easyblocks/h/hpcc.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/easybuild/easyblocks/h/hpcc.py b/easybuild/easyblocks/h/hpcc.py index 302305f129..5ef259f3c2 100644 --- a/easybuild/easyblocks/h/hpcc.py +++ b/easybuild/easyblocks/h/hpcc.py @@ -33,8 +33,6 @@ from easybuild.easyblocks.h.hpl import EB_HPL from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import change_dir, remove_file, symlink -from easybuild.tools.run import run_cmd class EB_HPCC(EB_HPL): From 4e2ed03189bcdd1b9c4bb5b99381ead4763432ba Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Fri, 29 Sep 2023 15:27:53 +0200 Subject: [PATCH 019/145] typo in comment --- easybuild/easyblocks/h/hpl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/h/hpl.py b/easybuild/easyblocks/h/hpl.py index b4066969dd..f88ca5d583 100644 --- a/easybuild/easyblocks/h/hpl.py +++ b/easybuild/easyblocks/h/hpl.py @@ -44,7 +44,7 @@ class EB_HPL(ConfigureMake): """ Support for building HPL (High Performance Linpack) - - creat Make.UNKNOWN + - create Make.UNKNOWN - build with make and install """ From 316f47ce1d8cf179a92c7125a03eef6df20f3cab Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Fri, 29 Sep 2023 15:29:15 +0200 Subject: [PATCH 020/145] add comment --- easybuild/easyblocks/h/hpl.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/easyblocks/h/hpl.py b/easybuild/easyblocks/h/hpl.py index f88ca5d583..7e34b30790 100644 --- a/easybuild/easyblocks/h/hpl.py +++ b/easybuild/easyblocks/h/hpl.py @@ -83,6 +83,7 @@ def configure_step(self, subdir=None): def build_step(self, topdir=None): """ Build with make and correct make options + - provide topdir argument so this can be reused in HPCC easyblock """ for envvar in ['MPICC', 'LIBLAPACK_MT', 'CPPFLAGS', 'LDFLAGS', 'CFLAGS']: From 891587e7553c270a283dffc8440b5c3891cc9361 Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Fri, 29 Sep 2023 15:30:53 +0200 Subject: [PATCH 021/145] remove stale comment --- easybuild/easyblocks/h/hpcc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/easybuild/easyblocks/h/hpcc.py b/easybuild/easyblocks/h/hpcc.py index 5ef259f3c2..8f11d5b924 100644 --- a/easybuild/easyblocks/h/hpcc.py +++ b/easybuild/easyblocks/h/hpcc.py @@ -45,7 +45,6 @@ class EB_HPCC(EB_HPL): def configure_step(self): """ Create Make.UNKNOWN file to build from - - provide subdir argument so this can be reused in HPCC easyblock """ super(EB_HPCC, self).configure_step(subdir='hpl') From 382a709b8bd01c2dbeb8f17ecd1080b7b9d2bece Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Fri, 29 Sep 2023 16:03:45 +0200 Subject: [PATCH 022/145] update hpl import --- easybuild/easyblocks/h/hpcc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/h/hpcc.py b/easybuild/easyblocks/h/hpcc.py index 8f11d5b924..6cf4fee09b 100644 --- a/easybuild/easyblocks/h/hpcc.py +++ b/easybuild/easyblocks/h/hpcc.py @@ -31,7 +31,7 @@ import os import shutil -from easybuild.easyblocks.h.hpl import EB_HPL +from easybuild.easyblocks.hpl import EB_HPL from easybuild.tools.build_log import EasyBuildError From 4b7f6fffbcf48cd913e64176723fca6c7552cbc9 Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Fri, 29 Sep 2023 16:38:21 +0200 Subject: [PATCH 023/145] also copy _hpccinf.txt to installdir --- easybuild/easyblocks/h/hpcc.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/h/hpcc.py b/easybuild/easyblocks/h/hpcc.py index 6cf4fee09b..2efd9ec759 100644 --- a/easybuild/easyblocks/h/hpcc.py +++ b/easybuild/easyblocks/h/hpcc.py @@ -60,11 +60,12 @@ def install_step(self): """ srcdir = self.cfg['start_dir'] destdir = os.path.join(self.installdir, 'bin') - filename = 'hpcc' + srcfile = None try: os.makedirs(destdir) - srcfile = os.path.join(srcdir, filename) - shutil.copy2(srcfile, destdir) + for filename in ["hpcc", "_hpccinf.txt"]: + srcfile = os.path.join(srcdir, filename) + shutil.copy2(srcfile, destdir) except OSError as err: raise EasyBuildError("Copying %s to installation dir %s failed: %s", srcfile, destdir, err) From 575d65cb45dd962ce6e2121b2a8cee00b77412ff Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Sat, 30 Sep 2023 16:25:26 +0200 Subject: [PATCH 024/145] add check for `-Dccflags` and add `preconfigopts` in perl easyblock --- easybuild/easyblocks/p/perl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/p/perl.py b/easybuild/easyblocks/p/perl.py index d96505ce7d..11542473cf 100644 --- a/easybuild/easyblocks/p/perl.py +++ b/easybuild/easyblocks/p/perl.py @@ -75,7 +75,7 @@ def configure_step(self): configopts = [ self.cfg['configopts'], '-Dcc="{0}"'.format(os.getenv('CC')), - '-Dccflags="{0}"'.format(os.getenv('CFLAGS')), + '-Dccflags="{0}"'.format(os.getenv('CFLAGS')) if '-Dccflags' not in self.cfg['configopts'] else '', '-Dinc_version_list=none', '-Dprefix=%(installdir)s', # guarantee that scripts are installed in /bin in the installation directory (and not in a guessed path) @@ -116,7 +116,7 @@ def configure_step(self): if os.getenv('COLUMNS', None) == '0': unset_env_vars(['COLUMNS']) - cmd = './Configure -de %s' % configopts + cmd = '%s ./Configure -de %s' % (self.cfg['preconfigopts'], configopts) run_cmd(cmd, log_all=True, simple=True) def test_step(self): From 706970eb05ac970d3435a71a5894d499fdc45ed4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 3 Oct 2023 09:05:40 +0200 Subject: [PATCH 025/145] enhance TensorFlow easyblock to avoid use of -mcpu=native for XNNPACK component when building on aarch64 --- easybuild/easyblocks/t/tensorflow.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/t/tensorflow.py b/easybuild/easyblocks/t/tensorflow.py index de9b97296a..926a1920cf 100644 --- a/easybuild/easyblocks/t/tensorflow.py +++ b/easybuild/easyblocks/t/tensorflow.py @@ -48,7 +48,7 @@ from easybuild.tools.filetools import is_readable, read_file, which, write_file, remove_file from easybuild.tools.modules import get_software_root, get_software_version, get_software_libdir from easybuild.tools.run import run_cmd -from easybuild.tools.systemtools import X86_64, get_cpu_architecture, get_os_name, get_os_version +from easybuild.tools.systemtools import AARCH64, X86_64, get_cpu_architecture, get_os_name, get_os_version CPU_DEVICE = 'cpu' @@ -687,6 +687,19 @@ def configure_step(self): cmd = self.cfg['preconfigopts'] + './configure ' + self.cfg['configopts'] run_cmd(cmd, log_all=True, simple=True) + # when building on Arm 64-bit we can't just use --copt=-mcpu=native (or likewise for any -mcpu=...), + # because it breaks the build of XNNPACK; + # see also https://github.com/easybuilders/easybuild-easyconfigs/issues/18899 + if get_cpu_architecture() == AARCH64: + tf_conf_bazelrc = os.path.join(self.start_dir, '.tf_configure.bazelrc') + regex_subs = [ + # use --per_file_copt instead of --copt to selectively use -mcpu=native (not for XNNPACK), + # the leading '-' ensures that -mcpu=native is *not* used when building XNNPACK; + # see https://github.com/google/XNNPACK/issues/5566 + https://bazel.build/docs/user-manual#per-file-copt + ('--copt=-mcpu=', '--per_file_copt=-.*XNNPACK/.*@-mcpu='), + ] + apply_regex_substitutions(tf_conf_bazelrc, regex_subs) + def patch_crosstool_files(self): """Patches the CROSSTOOL files to include EasyBuild provided compiler paths""" inc_paths, lib_paths = [], [] From 4e265c2618876eadf58da9c335d3f4d8b36c17e5 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 4 Oct 2023 09:02:18 +0200 Subject: [PATCH 026/145] only use -DCMAKE_SKIP_RPATH=ON for CMake < 3.5.0 --- easybuild/easyblocks/generic/cmakemake.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/generic/cmakemake.py b/easybuild/easyblocks/generic/cmakemake.py index 79235a644e..0401f872e4 100644 --- a/easybuild/easyblocks/generic/cmakemake.py +++ b/easybuild/easyblocks/generic/cmakemake.py @@ -267,9 +267,10 @@ def configure_step(self, srcdir=None, builddir=None): self.log.info("Using absolute path to compiler command: %s", value) options[option] = value - if build_option('rpath'): + if build_option('rpath') and LooseVersion(self.cmake_version) < LooseVersion('3.5.0'): # instruct CMake not to fiddle with RPATH when --rpath is used, since it will undo stuff on install... - # https://github.com/LLNL/spack/blob/0f6a5cd38538e8969d11bd2167f11060b1f53b43/lib/spack/spack/build_environment.py#L416 + # this is only required for CMake < 3.5.0, since newer version are more careful w.r.t. RPATH, + # see https://github.com/Kitware/CMake/commit/3ec9226779776811240bde88a3f173c29aa935b5 options['CMAKE_SKIP_RPATH'] = 'ON' # show what CMake is doing by default From 9dbb29979f6f48808e1c9df5bcdbcb09caec38b8 Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Sat, 7 Oct 2023 22:47:23 +0200 Subject: [PATCH 027/145] add more test programs to OpenMPI EasyBlock --- easybuild/easyblocks/o/openmpi.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/o/openmpi.py b/easybuild/easyblocks/o/openmpi.py index cf7c73151f..e76dd131bf 100644 --- a/easybuild/easyblocks/o/openmpi.py +++ b/easybuild/easyblocks/o/openmpi.py @@ -211,8 +211,18 @@ def sanity_check_step(self): mpi_cmd_tmpl, params = get_mpi_cmd_template(toolchain.OPENMPI, dict(), mpi_version=self.version) # Limit number of ranks to 8 to avoid it failing due to hyperthreading ranks = min(8, self.cfg['parallel']) - for src, compiler in (('hello_c.c', 'mpicc'), ('hello_mpifh.f', 'mpifort'), ('hello_usempi.f90', 'mpif90')): - src_path = os.path.join(self.cfg['start_dir'], 'examples', src) + for srcdir, src, compiler in ( + ('examples', 'hello_c.c', 'mpicc'), + ('examples', 'hello_mpifh.f', 'mpifort'), + ('examples', 'hello_usempi.f90', 'mpif90'), + ('examples', 'ring_c.c', 'mpicc'), + ('examples', 'ring_mpifh.f', 'mpifort'), + ('examples', 'ring_usempi.f90', 'mpif90'), + ('test/simple', 'thread_init.c', 'mpicc'), + ('test/simple', 'intercomm1.c', 'mpicc'), + ('test/simple', 'mpi_barrier.c', 'mpicc'), + ): + src_path = os.path.join(self.cfg['start_dir'], srcdir, src) if os.path.exists(src_path): test_exe = os.path.join(self.builddir, 'mpi_test_' + os.path.splitext(src)[0]) self.log.info("Adding minimal MPI test program to sanity checks: %s", test_exe) From 78f71dc9a60fc092cf8c5a9a5a65c656c6b2c02f Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 16 Oct 2023 12:42:29 +0200 Subject: [PATCH 028/145] Use dependencies instead of `get_software_root` to determine plugins Allows working with --sanity-check-only where the modules may not be loaded yet --- easybuild/easyblocks/u/ucx_plugins.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/easybuild/easyblocks/u/ucx_plugins.py b/easybuild/easyblocks/u/ucx_plugins.py index 2ece31b019..a80aa379f5 100644 --- a/easybuild/easyblocks/u/ucx_plugins.py +++ b/easybuild/easyblocks/u/ucx_plugins.py @@ -54,16 +54,16 @@ def plugins(self): """Property to determine list of plugins based on loaded dependencies, or return cached list of plugins.""" if self._plugins is None: plugins = defaultdict(list) + dep_names = set(dep['name'] for dep in self.cfg.dependencies()) - if get_software_root('CUDAcore') or get_software_root('CUDA'): + if 'CUDAcore' in dep_names or 'CUDA' in dep_names: for key in ('ucm', 'uct', 'ucx_perftest'): plugins[key].append('cuda') - if get_software_root('GDRCopy'): + if 'GDRCopy' in dep_names: plugins['uct_cuda'].append('gdrcopy') - # To be supported in the future: - if get_software_root('ROCm'): + if 'ROCm' in dep_names: for key in ('ucm', 'uct', 'ucx_perftest'): plugins[key].append('rocm') @@ -94,7 +94,6 @@ def configure_step(self): self.makefile_dirs.extend(os.path.join(x, 'cuda') for x in ('uct', 'ucm', 'tools/perf')) - # To be supported in the future: rocmroot = get_software_root('ROCm') if rocmroot: configopts += '--with-rocm=%s ' % rocmroot From 97a1194c572024515b4bec859b3419f80e8448f0 Mon Sep 17 00:00:00 2001 From: Viktor Rehnberg Date: Tue, 17 Oct 2023 08:40:15 +0200 Subject: [PATCH 029/145] adding easyblocks: palm.py --- easybuild/easyblocks/p/palm.py | 90 ++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 easybuild/easyblocks/p/palm.py diff --git a/easybuild/easyblocks/p/palm.py b/easybuild/easyblocks/p/palm.py new file mode 100644 index 0000000000..213001b1f2 --- /dev/null +++ b/easybuild/easyblocks/p/palm.py @@ -0,0 +1,90 @@ +## +# Copyright 2018-2019 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for building and installing PALM, implemented as an easyblock + +@author: Viktor Rehnberg (Chalmers University of Technology) +""" +import os +import tempfile + +import easybuild.tools.environment as env +from easybuild.framework.easyblock import EasyBlock +from easybuild.tools.config import build_option +from easybuild.tools.filetools import find_glob_pattern +from easybuild.tools.run import run_cmd + + +class EB_PALM(EasyBlock): + """Support for building/installing PALM.""" + + def __init__(self, *args, **kwargs): + """Initialise PALM easyblock.""" + super().__init__(*args, **kwargs) + + def configure_step(self): + """No configuration procedure for PALM.""" + pass + + def build_step(self): + """No build procedure for PALM.""" + pass + + def install_step(self): + """Custom install procedure for PALM.""" + + install_script_pattern = "install" + if self.dry_run: + install_script = install_script_pattern + else: + install_script = find_glob_pattern(install_script_pattern) + + cmd = ' '.join([ + self.cfg['preinstallopts'], + "bash", + install_script, + "-p %s" % self.installdir, + self.cfg['installopts'], + ]) + run_cmd(cmd, log_all=True, simple=True) + + def sanity_check_step(self): + """Custom sanity check for PALM.""" + custom_paths = { + 'files': [os.path.join(self.installdir, 'bin', 'palmrun')], + 'dirs': [], + } + super().sanity_check_step(custom_paths=custom_paths) + + def make_module_extra(self): + """Extra statements specific to PALM to include in generated module file.""" + txt = super().make_module_extra() + + bin_dirs = [ + os.path.join(self.installdir, 'bin'), + ] + txt += self.module_generator.prepend_paths('PATH', bin_dirs) + + return txt From f7ab7bae79f8f1747a1d5122295e5da8fdb3d1e2 Mon Sep 17 00:00:00 2001 From: Viktor Rehnberg Date: Tue, 17 Oct 2023 08:42:16 +0200 Subject: [PATCH 030/145] Remove unused imports --- easybuild/easyblocks/p/palm.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/easybuild/easyblocks/p/palm.py b/easybuild/easyblocks/p/palm.py index 213001b1f2..677aefaa94 100644 --- a/easybuild/easyblocks/p/palm.py +++ b/easybuild/easyblocks/p/palm.py @@ -28,11 +28,8 @@ @author: Viktor Rehnberg (Chalmers University of Technology) """ import os -import tempfile -import easybuild.tools.environment as env from easybuild.framework.easyblock import EasyBlock -from easybuild.tools.config import build_option from easybuild.tools.filetools import find_glob_pattern from easybuild.tools.run import run_cmd From b0c1dfb7418999580dd493622b45a5aeda5709fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Tue, 17 Oct 2023 19:47:32 +0200 Subject: [PATCH 031/145] Switch to python2 and 3 compatible kwarg --- easybuild/easyblocks/generic/bundle.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/easybuild/easyblocks/generic/bundle.py b/easybuild/easyblocks/generic/bundle.py index 8ae2432913..2631f0f62e 100644 --- a/easybuild/easyblocks/generic/bundle.py +++ b/easybuild/easyblocks/generic/bundle.py @@ -66,11 +66,6 @@ def extra_options(extra_vars=None): return EasyBlock.extra_options(extra_vars) def __init__(self, *args, **kwargs): - """ - Constructor for Bundle easyblock - """ - # add check_for_sources named argument in a Python 2+3 compatible way - check_for_sources = kwargs.get('check_for_sources', True) """Initialize easyblock.""" super(Bundle, self).__init__(*args, **kwargs) self.altroot = None @@ -82,7 +77,8 @@ def __init__(self, *args, **kwargs): # list of EasyConfig instances of components for which to run sanity checks self.comp_cfgs_sanity_check = [] - # list of sources for bundle itself *must* be empty + check_for_sources = check_for_sources = kwargs.get('check_for_sources', True) + # list of sources for bundle itself *must* be empty (unless overridden by subclass) if check_for_sources: if self.cfg['sources']: raise EasyBuildError("List of sources for bundle itself must be empty, found %s", self.cfg['sources']) From 5e53aba741b07648d8fc676b79068d9ae954237b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Tue, 17 Oct 2023 20:00:46 +0200 Subject: [PATCH 032/145] Fix copy-paste typo --- easybuild/easyblocks/generic/bundle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/bundle.py b/easybuild/easyblocks/generic/bundle.py index 2631f0f62e..f52b575254 100644 --- a/easybuild/easyblocks/generic/bundle.py +++ b/easybuild/easyblocks/generic/bundle.py @@ -77,7 +77,7 @@ def __init__(self, *args, **kwargs): # list of EasyConfig instances of components for which to run sanity checks self.comp_cfgs_sanity_check = [] - check_for_sources = check_for_sources = kwargs.get('check_for_sources', True) + check_for_sources = kwargs.get('check_for_sources', True) # list of sources for bundle itself *must* be empty (unless overridden by subclass) if check_for_sources: if self.cfg['sources']: From 572cde91a9bd3979ae17a00a1f49c6e32299062e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Tue, 17 Oct 2023 22:08:08 +0200 Subject: [PATCH 033/145] Split super init calls --- easybuild/easyblocks/generic/cargopythonbundle.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/cargopythonbundle.py b/easybuild/easyblocks/generic/cargopythonbundle.py index 2cbec2dd94..a55602770f 100644 --- a/easybuild/easyblocks/generic/cargopythonbundle.py +++ b/easybuild/easyblocks/generic/cargopythonbundle.py @@ -50,7 +50,8 @@ def extra_options(extra_vars=None): def __init__(self, *args, **kwargs): """Constructor for CargoPythonBundle easyblock.""" - super(CargoPythonBundle, self).__init__(*args, check_for_sources=False, **kwargs) + PythonBundle.__init__(self, *args, check_for_sources=False, **kwargs) + Cargo.__init__(self, *args, **kwargs) def extract_step(self): """Specifically use the overloaded variant from Cargo as is populates vendored sources with checksums.""" From 7599bdea675807dc91f5f49882cc08b459dfb716 Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Wed, 18 Oct 2023 14:59:13 +0200 Subject: [PATCH 034/145] fix CMAKE_PREFIX_PATH in mkl easyblock --- easybuild/easyblocks/i/imkl.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easybuild/easyblocks/i/imkl.py b/easybuild/easyblocks/i/imkl.py index 978b42137c..a342a4686c 100644 --- a/easybuild/easyblocks/i/imkl.py +++ b/easybuild/easyblocks/i/imkl.py @@ -546,11 +546,13 @@ def make_module_req_guess(self): os.path.join(self.mkl_basedir, 'include'), os.path.join(self.mkl_basedir, 'include', 'fftw'), ] + cmake_prefix_path = [self.mkl_basedir] guesses.update({ 'PATH': [], 'LD_LIBRARY_PATH': library_path, 'LIBRARY_PATH': library_path, 'CPATH': cpath, + 'CMAKE_PREFIX_PATH': cmake_prefix_path, 'PKG_CONFIG_PATH': pkg_config_path, }) if self.cfg['flexiblas']: From 7fc1e6749518f6ffee78a29de62899c98d9e41c3 Mon Sep 17 00:00:00 2001 From: Viktor Rehnberg <35767167+VRehnberg@users.noreply.github.com> Date: Mon, 23 Oct 2023 09:53:18 +0200 Subject: [PATCH 035/145] Simplify make_module_extra Remove self.installdir from bin_dir paths on akesa's suggestion https://github.com/easybuilders/easybuild-easyblocks/pull/3020#discussion_r1368207994 --- easybuild/easyblocks/p/palm.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/easybuild/easyblocks/p/palm.py b/easybuild/easyblocks/p/palm.py index 677aefaa94..f83e9bd2a8 100644 --- a/easybuild/easyblocks/p/palm.py +++ b/easybuild/easyblocks/p/palm.py @@ -79,9 +79,6 @@ def make_module_extra(self): """Extra statements specific to PALM to include in generated module file.""" txt = super().make_module_extra() - bin_dirs = [ - os.path.join(self.installdir, 'bin'), - ] - txt += self.module_generator.prepend_paths('PATH', bin_dirs) + txt += self.module_generator.prepend_paths('PATH', ['bin']) return txt From df832a74ef4897453b6a1fca428cc6ad6e6a553d Mon Sep 17 00:00:00 2001 From: sassy Date: Mon, 23 Oct 2023 21:01:26 +0100 Subject: [PATCH 036/145] Sanity-test expanded to 4.2.0 version --- easybuild/easyblocks/w/wxpython.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easybuild/easyblocks/w/wxpython.py b/easybuild/easyblocks/w/wxpython.py index 35b22b1867..939e2718a1 100644 --- a/easybuild/easyblocks/w/wxpython.py +++ b/easybuild/easyblocks/w/wxpython.py @@ -127,6 +127,9 @@ def sanity_check_step(self): files.extend([os.path.join('bin', 'wxrc')]) dirs.extend(['include', 'share']) py_bins.extend(['alacarte', 'alamode', 'wrap']) + elif LooseVersion(self.version) >= LooseVersion("4.2"): + majver = '3.2' # this is 3.2 in ver 4.2.x + py_bins.extend(['slices', 'slicesshell']) elif LooseVersion(self.version) >= LooseVersion("4.1"): majver = '3.1' # this is 3.1 in ver 4.1.x py_bins.extend(['slices', 'slicesshell']) From b73d7887fab1aa959a4d28190cbba22c427ad5eb Mon Sep 17 00:00:00 2001 From: Jakob Schiotz Date: Tue, 24 Oct 2023 12:39:27 +0200 Subject: [PATCH 037/145] Fix the blas/lapack name passed to meson when building scipy with MKL. --- easybuild/easyblocks/s/scipy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/s/scipy.py b/easybuild/easyblocks/s/scipy.py index fa82d0a866..094a0259b5 100644 --- a/easybuild/easyblocks/s/scipy.py +++ b/easybuild/easyblocks/s/scipy.py @@ -124,7 +124,7 @@ def configure_step(self): if lapack_lib == toolchain.FLEXIBLAS: blas_lapack = 'flexiblas' elif lapack_lib == toolchain.INTELMKL: - blas_lapack = 'mkl' + blas_lapack = 'mkl-dynamic-lp64-seq' elif lapack_lib == toolchain.OPENBLAS: blas_lapack = 'openblas' else: From 56dbb5d226215e98ee8fd449a80b18c83b339303 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 27 Oct 2023 12:14:39 +0200 Subject: [PATCH 038/145] use dependency_names method in UCX-plugins easyblock --- easybuild/easyblocks/u/ucx_plugins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/u/ucx_plugins.py b/easybuild/easyblocks/u/ucx_plugins.py index a80aa379f5..a00512c48c 100644 --- a/easybuild/easyblocks/u/ucx_plugins.py +++ b/easybuild/easyblocks/u/ucx_plugins.py @@ -54,7 +54,7 @@ def plugins(self): """Property to determine list of plugins based on loaded dependencies, or return cached list of plugins.""" if self._plugins is None: plugins = defaultdict(list) - dep_names = set(dep['name'] for dep in self.cfg.dependencies()) + dep_names = self.cfg.dependency_names() if 'CUDAcore' in dep_names or 'CUDA' in dep_names: for key in ('ucm', 'uct', 'ucx_perftest'): From 25792946369d8a7b27a1108723eb81df425f9b42 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Sat, 28 Oct 2023 16:38:26 +0800 Subject: [PATCH 039/145] prepare release notes for EasyBuild v4.8.2 + bump version to 4.8.2 --- RELEASE_NOTES | 26 ++++++++++++++++++++++++++ easybuild/easyblocks/__init__.py | 2 +- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 8dd90e0374..bfa2d4316b 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -6,6 +6,32 @@ These release notes can also be consulted at http://easybuild.readthedocs.org/en The latest version of easybuild-easyblocks provides 251 software-specific easyblocks and 42 generic easyblocks. +v4.8.2 (29 October 2023) +------------------------ + +update/bugfix release + +- minor enhancements and updates, including: + - allow use of `test_cmd` without `runtest` for `ConfigureMake` (#2837) + - enhance `CMakeMake` easyblock to run `ctest` command if `runtest` is `True` (#2838) + - enhance `Conda` easyblock: add support for using custom `conda` command (like `mamba`), and clean up after installation (#2996) + - update WRF easyblock to correctly determine `WRF*` subdirectory for WRF versions >= 4.5.1 (#2997) + - update numpy easyblock to be able to use FlexiBLAS with MKL as dependency (#2999) + - add error regexp for PyTorch tests run in subprocess and enhance error message (#3003) + - don't blindly overwrite `-Dccflags` + honour `preconfigopts` in Perl easyblock (#3010) + - use more test programs in sanity check step of OpenMPI easyblock (#3016) + - fix CMAKE_PREFIX_PATH in mkl easyblock (#3021) + - update sanity check in wxPython easyblock for version 4.2.0 (#3023) +- various bug fixes, including: + - fix creation of symlink for libraries in TBB easyblock (#2927) + - remove stray newline in pytorch error message (#3005) + - fix `--sanity-check-only` and `--module-only` for UCX plugins (#3007) + - replace umlaut in docstring of PerlBundle easyblock (to fix Non-ASCII character error when using Python 2.7) (#3008) + - enhance TensorFlow easyblock to avoid use of `-mcpu=native` for XNNPACK component when building on aarch64 (#3011) + - only use `-DCMAKE_SKIP_RPATH=ON` for CMake < 3.5.0 (#3012) + - fix the blas/lapack name passed to meson when building recent scipy versions (>= 1.9.0) on top of Intel MKL (#3024) + + v4.8.1 (11 September 2023) -------------------------- diff --git a/easybuild/easyblocks/__init__.py b/easybuild/easyblocks/__init__.py index 1103ef43cf..7d1118ed84 100644 --- a/easybuild/easyblocks/__init__.py +++ b/easybuild/easyblocks/__init__.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.8.2.dev0') +VERSION = LooseVersion('4.8.2') UNKNOWN = 'UNKNOWN' From bcc801401f8642438d7c9ac33d2d1dcb78833b7d Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Sat, 28 Oct 2023 17:08:32 +0200 Subject: [PATCH 040/145] make suggested changes --- easybuild/easyblocks/h/hpcc.py | 18 ++++++++---------- easybuild/easyblocks/h/hpl.py | 21 +++++++-------------- 2 files changed, 15 insertions(+), 24 deletions(-) diff --git a/easybuild/easyblocks/h/hpcc.py b/easybuild/easyblocks/h/hpcc.py index 2efd9ec759..192da6919c 100644 --- a/easybuild/easyblocks/h/hpcc.py +++ b/easybuild/easyblocks/h/hpcc.py @@ -29,10 +29,9 @@ """ import os -import shutil from easybuild.easyblocks.hpl import EB_HPL -from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.filetools import copy_file, mkdir class EB_HPCC(EB_HPL): @@ -61,13 +60,10 @@ def install_step(self): srcdir = self.cfg['start_dir'] destdir = os.path.join(self.installdir, 'bin') srcfile = None - try: - os.makedirs(destdir) - for filename in ["hpcc", "_hpccinf.txt"]: - srcfile = os.path.join(srcdir, filename) - shutil.copy2(srcfile, destdir) - except OSError as err: - raise EasyBuildError("Copying %s to installation dir %s failed: %s", srcfile, destdir, err) + mkdir(destdir) + for filename in ["hpcc", "_hpccinf.txt"]: + srcfile = os.path.join(srcdir, filename) + copy_file(srcfile, destdir) def sanity_check_step(self): """ @@ -79,4 +75,6 @@ def sanity_check_step(self): 'dirs': [] } - super(EB_HPL, self).sanity_check_step(custom_paths) + custom_commands = ['hpcc'] + + super(EB_HPL, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) diff --git a/easybuild/easyblocks/h/hpl.py b/easybuild/easyblocks/h/hpl.py index 7e34b30790..999da4ab77 100644 --- a/easybuild/easyblocks/h/hpl.py +++ b/easybuild/easyblocks/h/hpl.py @@ -33,11 +33,10 @@ """ import os -import shutil from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import change_dir, remove_file, symlink +from easybuild.tools.filetools import change_dir, copy_file, mkdir, remove_file, symlink from easybuild.tools.run import run_cmd @@ -62,10 +61,7 @@ def configure_step(self, subdir=None): makeincfile = os.path.join(basedir, 'Make.UNKNOWN') setupdir = os.path.join(basedir, 'setup') - try: - change_dir(setupdir) - except OSError as err: - raise EasyBuildError("Failed to change to to dir %s: %s", setupdir, err) + change_dir(setupdir) cmd = "/bin/bash make_generic" @@ -73,7 +69,7 @@ def configure_step(self, subdir=None): try: remove_file(os.path.join(makeincfile)) - symlink(os.path.join(setupdir, 'Make.UNKNOWN'), os.path.join(makeincfile)) + symlink(os.path.join(setupdir, 'Make.UNKNOWN'), makeincfile) except OSError as err: raise EasyBuildError("Failed to symlink Make.UNKNOWN from %s to %s: %s", setupdir, makeincfile, err) @@ -122,13 +118,10 @@ def install_step(self): srcdir = os.path.join(self.cfg['start_dir'], 'bin', 'UNKNOWN') destdir = os.path.join(self.installdir, 'bin') srcfile = None - try: - os.makedirs(destdir) - for filename in ["xhpl", "HPL.dat"]: - srcfile = os.path.join(srcdir, filename) - shutil.copy2(srcfile, destdir) - except OSError as err: - raise EasyBuildError("Copying %s to installation dir %s failed: %s", srcfile, destdir, err) + mkdir(destdir) + for filename in ["xhpl", "HPL.dat"]: + srcfile = os.path.join(srcdir, filename) + copy_file(srcfile, destdir) def sanity_check_step(self): """ From 68e55ac5a1220db869e993c47fe0bf7d38bd3a76 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 28 Oct 2023 17:12:34 +0200 Subject: [PATCH 041/145] tweak release notes for EasyBuild v4.8.2 --- RELEASE_NOTES | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index bfa2d4316b..027e4a47e3 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -17,14 +17,13 @@ update/bugfix release - enhance `Conda` easyblock: add support for using custom `conda` command (like `mamba`), and clean up after installation (#2996) - update WRF easyblock to correctly determine `WRF*` subdirectory for WRF versions >= 4.5.1 (#2997) - update numpy easyblock to be able to use FlexiBLAS with MKL as dependency (#2999) - - add error regexp for PyTorch tests run in subprocess and enhance error message (#3003) + - add error regexp for PyTorch tests run in subprocess and enhance error message (#3003, #3005) - don't blindly overwrite `-Dccflags` + honour `preconfigopts` in Perl easyblock (#3010) - use more test programs in sanity check step of OpenMPI easyblock (#3016) - - fix CMAKE_PREFIX_PATH in mkl easyblock (#3021) + - fix `CMAKE_PREFIX_PATH` in imkl easyblock (#3021) - update sanity check in wxPython easyblock for version 4.2.0 (#3023) - various bug fixes, including: - fix creation of symlink for libraries in TBB easyblock (#2927) - - remove stray newline in pytorch error message (#3005) - fix `--sanity-check-only` and `--module-only` for UCX plugins (#3007) - replace umlaut in docstring of PerlBundle easyblock (to fix Non-ASCII character error when using Python 2.7) (#3008) - enhance TensorFlow easyblock to avoid use of `-mcpu=native` for XNNPACK component when building on aarch64 (#3011) From 1cf46f443f26dfc6d08896e16bf97e3bb5da742d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Sun, 29 Oct 2023 00:42:12 +0200 Subject: [PATCH 042/145] Allow version mismatch and include versionsuffix in modulerc --- easybuild/easyblocks/generic/modulerc.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/generic/modulerc.py b/easybuild/easyblocks/generic/modulerc.py index aa67e90fe3..c6bd4b988c 100644 --- a/easybuild/easyblocks/generic/modulerc.py +++ b/easybuild/easyblocks/generic/modulerc.py @@ -30,6 +30,7 @@ import os from easybuild.framework.easyblock import EasyBlock +from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.easyconfig.easyconfig import ActiveMNS from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.config import install_path @@ -41,6 +42,16 @@ class ModuleRC(EasyBlock): Generic easyblock to create a software-specific .modulerc file """ + @staticmethod + def extra_options(extra_vars=None): + """Define extra easyconfig parameters specific to ModuleRC""" + if extra_vars is None: + extra_vars = {} + extra_vars.update({ + 'check_version': [True, "Check version is prefix of dependency", CUSTOM], + }) + return EasyBlock.extra_options(extra_vars) + def configure_step(self): """Do nothing.""" pass @@ -67,7 +78,8 @@ def make_module_step(self, fake=False): raise EasyBuildError("Name does not match dependency name: %s vs %s", self.name, deps[0]['name']) # ensure version to alias to is a prefix of the version of the dependency - if not deps[0]['version'].startswith(self.version) and not self.version == "default": + if self.cfg['check_version'] and \ + not deps[0]['version'].startswith(self.version) and not self.version == "default": raise EasyBuildError("Version is not 'default' and not a prefix of dependency version: %s vs %s", self.version, deps[0]['version']) @@ -85,7 +97,7 @@ def make_module_step(self, fake=False): module_version_specs = { 'modname': alias_modname, - 'sym_version': self.version, + 'sym_version': self.version + self.cfg['versionsuffix'], 'version': deps[0]['version'], } self.module_generator.modulerc(module_version=module_version_specs, filepath=modulerc) From 3015baacbff54f78a5eba54c6ef591fd04e2b2bd Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Sun, 29 Oct 2023 07:55:35 +0100 Subject: [PATCH 043/145] more suggested changes --- easybuild/easyblocks/h/hpl.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/easybuild/easyblocks/h/hpl.py b/easybuild/easyblocks/h/hpl.py index 999da4ab77..2508d74341 100644 --- a/easybuild/easyblocks/h/hpl.py +++ b/easybuild/easyblocks/h/hpl.py @@ -67,11 +67,8 @@ def configure_step(self, subdir=None): run_cmd(cmd, log_all=True, simple=True, log_output=True) - try: - remove_file(os.path.join(makeincfile)) - symlink(os.path.join(setupdir, 'Make.UNKNOWN'), makeincfile) - except OSError as err: - raise EasyBuildError("Failed to symlink Make.UNKNOWN from %s to %s: %s", setupdir, makeincfile, err) + remove_file(makeincfile) + symlink(os.path.join(setupdir, 'Make.UNKNOWN'), makeincfile) # go back change_dir(self.cfg['start_dir']) From 6a71006f4c7970db2a8154b775e16b84d8075735 Mon Sep 17 00:00:00 2001 From: Viktor Rehnberg <35767167+VRehnberg@users.noreply.github.com> Date: Mon, 30 Oct 2023 10:06:29 +0100 Subject: [PATCH 044/145] Remove make_module_extra as requested --- easybuild/easyblocks/p/palm.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/easybuild/easyblocks/p/palm.py b/easybuild/easyblocks/p/palm.py index f83e9bd2a8..f0777a4dbe 100644 --- a/easybuild/easyblocks/p/palm.py +++ b/easybuild/easyblocks/p/palm.py @@ -75,10 +75,4 @@ def sanity_check_step(self): } super().sanity_check_step(custom_paths=custom_paths) - def make_module_extra(self): - """Extra statements specific to PALM to include in generated module file.""" - txt = super().make_module_extra() - - txt += self.module_generator.prepend_paths('PATH', ['bin']) - return txt From 6e367e9adb77c4aa5c460ff500fb4c746a91ba82 Mon Sep 17 00:00:00 2001 From: Viktor Rehnberg <35767167+VRehnberg@users.noreply.github.com> Date: Mon, 30 Oct 2023 10:07:39 +0100 Subject: [PATCH 045/145] Remove line missed in last commit --- easybuild/easyblocks/p/palm.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/easybuild/easyblocks/p/palm.py b/easybuild/easyblocks/p/palm.py index f0777a4dbe..de1af55341 100644 --- a/easybuild/easyblocks/p/palm.py +++ b/easybuild/easyblocks/p/palm.py @@ -74,5 +74,3 @@ def sanity_check_step(self): 'dirs': [], } super().sanity_check_step(custom_paths=custom_paths) - - return txt From f037c2a889c6fae4b1d64e6542b0b90d5525e3ed Mon Sep 17 00:00:00 2001 From: Viktor Rehnberg <35767167+VRehnberg@users.noreply.github.com> Date: Mon, 30 Oct 2023 10:52:17 +0100 Subject: [PATCH 046/145] Set copy right to 2023 --- easybuild/easyblocks/p/palm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/p/palm.py b/easybuild/easyblocks/p/palm.py index de1af55341..6674be5c7a 100644 --- a/easybuild/easyblocks/p/palm.py +++ b/easybuild/easyblocks/p/palm.py @@ -1,5 +1,5 @@ ## -# Copyright 2018-2019 Ghent University +# Copyright 2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), From 5f60a3dc3818953b99db873a5892f29ea57c1f89 Mon Sep 17 00:00:00 2001 From: appolloford Date: Tue, 31 Oct 2023 15:32:55 +0100 Subject: [PATCH 047/145] Simplify `if` statement --- easybuild/easyblocks/s/suitesparse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/s/suitesparse.py b/easybuild/easyblocks/s/suitesparse.py index 75812024f9..5960f39718 100644 --- a/easybuild/easyblocks/s/suitesparse.py +++ b/easybuild/easyblocks/s/suitesparse.py @@ -92,7 +92,7 @@ def configure_step(self): if parmetis or metis: if parmetis: metis_name = 'ParMETIS' - elif metis: + else: metis_name = 'METIS' metis_path = get_software_root(metis_name) metis_include = os.path.join(metis_path, 'include') From 4d4bc91f06bbb6a2ea47aa3040a47278d27519be Mon Sep 17 00:00:00 2001 From: appolloford Date: Wed, 1 Nov 2023 14:04:44 +0100 Subject: [PATCH 048/145] Remove unused imports --- easybuild/easyblocks/s/suitesparse.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/easybuild/easyblocks/s/suitesparse.py b/easybuild/easyblocks/s/suitesparse.py index 5960f39718..7afd352080 100644 --- a/easybuild/easyblocks/s/suitesparse.py +++ b/easybuild/easyblocks/s/suitesparse.py @@ -35,14 +35,12 @@ import fileinput import re import os -import shutil import sys -import stat from distutils.version import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import mkdir, write_file, adjust_permissions, copy_dir +from easybuild.tools.filetools import mkdir, write_file from easybuild.tools.modules import get_software_root from easybuild.tools.modules import get_software_libdir from easybuild.tools.systemtools import get_shared_lib_ext From da4f49e189dcf4f1d207050a6884ea81634a8aa1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 4 Nov 2023 21:08:21 +0100 Subject: [PATCH 049/145] bump version to 4.9.0dev --- easybuild/easyblocks/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/__init__.py b/easybuild/easyblocks/__init__.py index 7d1118ed84..115bad70c1 100644 --- a/easybuild/easyblocks/__init__.py +++ b/easybuild/easyblocks/__init__.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.8.2') +VERSION = LooseVersion('4.9.0.dev0') UNKNOWN = 'UNKNOWN' From 33ea1af1bc0343f26a7f3cde2318b0c3a28e9252 Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Wed, 8 Nov 2023 10:08:09 +0100 Subject: [PATCH 050/145] more more suggested changes --- easybuild/easyblocks/h/hpcc.py | 5 +++-- easybuild/easyblocks/h/hpl.py | 1 - 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/h/hpcc.py b/easybuild/easyblocks/h/hpcc.py index 192da6919c..57893788b2 100644 --- a/easybuild/easyblocks/h/hpcc.py +++ b/easybuild/easyblocks/h/hpcc.py @@ -45,12 +45,14 @@ def configure_step(self): """ Create Make.UNKNOWN file to build from """ + # the build script file should be created in the hpl subdir super(EB_HPCC, self).configure_step(subdir='hpl') def build_step(self): """ Build with make and correct make options """ + # TOPdir should always be ../../.. regardless of what it was in the HPL build script file super(EB_HPCC, self).build_step(topdir='../../..') def install_step(self): @@ -59,7 +61,6 @@ def install_step(self): """ srcdir = self.cfg['start_dir'] destdir = os.path.join(self.installdir, 'bin') - srcfile = None mkdir(destdir) for filename in ["hpcc", "_hpccinf.txt"]: srcfile = os.path.join(srcdir, filename) @@ -71,7 +72,7 @@ def sanity_check_step(self): """ custom_paths = { - 'files': ["bin/hpcc"], + 'files': ['bin/hpcc', 'bin/_hpccinf.txt'], 'dirs': [] } diff --git a/easybuild/easyblocks/h/hpl.py b/easybuild/easyblocks/h/hpl.py index 2508d74341..3599b3004a 100644 --- a/easybuild/easyblocks/h/hpl.py +++ b/easybuild/easyblocks/h/hpl.py @@ -114,7 +114,6 @@ def install_step(self): """ srcdir = os.path.join(self.cfg['start_dir'], 'bin', 'UNKNOWN') destdir = os.path.join(self.installdir, 'bin') - srcfile = None mkdir(destdir) for filename in ["xhpl", "HPL.dat"]: srcfile = os.path.join(srcdir, filename) From 55a3b6436430b0b9515bde6676f5c22a02c7aec4 Mon Sep 17 00:00:00 2001 From: Chia-Jung Hsu Date: Wed, 8 Nov 2023 16:07:50 +0100 Subject: [PATCH 051/145] Add an extra_option of `cmake_options` --- easybuild/easyblocks/s/suitesparse.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/s/suitesparse.py b/easybuild/easyblocks/s/suitesparse.py index 7afd352080..a4bd5be038 100644 --- a/easybuild/easyblocks/s/suitesparse.py +++ b/easybuild/easyblocks/s/suitesparse.py @@ -39,6 +39,7 @@ from distutils.version import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake +from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import mkdir, write_file from easybuild.tools.modules import get_software_root @@ -49,6 +50,14 @@ class EB_SuiteSparse(ConfigureMake): """Support for building SuiteSparse.""" + @staticmethod + def extra_options(extra_vars=None): + """Define extra easyconfig parameters""" + extra_vars = { + 'cmake_options': ['-DBLA_VENDOR=FlexiBLAS', "CMAKE_OPTIONS used by SuiteSparse since v6.0", CUSTOM], + } + return ConfigureMake.extra_options(extra_vars) + def __init__(self, *args, **kwargs): """Custom constructor for SuiteSparse easyblock, initialize custom class parameters.""" super(EB_SuiteSparse, self).__init__(*args, **kwargs) @@ -172,8 +181,11 @@ def configure_step(self): else: # after v6.0.0, no option for metis, its own metis is used anyway - # nothing to do here, set the CMAKE_OPTIONS in easyconfigs - pass + # set CMAKE_OPTIONS if it is not specified in easyconfigs + # CMAKE_INSTALL_PREFIX is managed by easybuild + base_cmake_options = '-DCMAKE_INSTALL_PREFIX=%s' % self.installdir + cmake_options = " ".join([base_cmake_options, self.cfg['cmake_options']]) + self.cfg.update('preinstallopts', 'CMAKE_OPTIONS="%s"' % cmake_options) def install_step(self): """Install by copying the contents of the builddir to the installdir (preserving permissions)""" From 99e97521f393e844f7ea4a9a6fd9e7dab1654d6d Mon Sep 17 00:00:00 2001 From: Chia-Jung Hsu Date: Tue, 14 Nov 2023 17:29:02 +0100 Subject: [PATCH 052/145] Add dependencies check for cmake and BLAS --- easybuild/easyblocks/s/suitesparse.py | 29 ++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/easybuild/easyblocks/s/suitesparse.py b/easybuild/easyblocks/s/suitesparse.py index a4bd5be038..6c9d27a235 100644 --- a/easybuild/easyblocks/s/suitesparse.py +++ b/easybuild/easyblocks/s/suitesparse.py @@ -54,7 +54,7 @@ class EB_SuiteSparse(ConfigureMake): def extra_options(extra_vars=None): """Define extra easyconfig parameters""" extra_vars = { - 'cmake_options': ['-DBLA_VENDOR=FlexiBLAS', "CMAKE_OPTIONS used by SuiteSparse since v6.0", CUSTOM], + 'cmake_options': ['', "CMAKE_OPTIONS used by SuiteSparse since v6.0", CUSTOM], } return ConfigureMake.extra_options(extra_vars) @@ -84,6 +84,11 @@ def configure_step(self): 'LAPACK': os.getenv('LIBLAPACK_MT'), } + cmake = get_software_root('CMake') + if not cmake and LooseVersion(self.version) >= LooseVersion('5.1.2'): + # graphblas exists from v5.1.2, needs cmake + raise EasyBuildError("CMake module is not loaded") + # Get CUDA and set it up appropriately cuda = get_software_root('CUDA') if cuda: @@ -163,9 +168,7 @@ def configure_step(self): self.cfg.update('installopts', 'LAPACK="%s"' % cfgvars.get('LAPACK')) if LooseVersion(self.version) >= LooseVersion('5.1.2'): - # graphblas exists, needs cmake - # v5.0.0 until v5.1.2 has no CMAKE_OPTIONS to set - # probably need patch + # v5.0.0 until v5.1.2 has no CMAKE_OPTIONS to set, patches are needed self.cfg.update('preinstallopts', 'CMAKE_OPTIONS="-DCMAKE_INSTALL_PREFIX=%s"' % self.installdir) # set METIS library @@ -183,9 +186,21 @@ def configure_step(self): # after v6.0.0, no option for metis, its own metis is used anyway # set CMAKE_OPTIONS if it is not specified in easyconfigs # CMAKE_INSTALL_PREFIX is managed by easybuild - base_cmake_options = '-DCMAKE_INSTALL_PREFIX=%s' % self.installdir - cmake_options = " ".join([base_cmake_options, self.cfg['cmake_options']]) - self.cfg.update('preinstallopts', 'CMAKE_OPTIONS="%s"' % cmake_options) + cmake_options = '-DCMAKE_INSTALL_PREFIX=%s' % self.installdir + + flexiblas = get_software_root('FlexiBLAS') + mkl = get_software_root('imkl') + openblas = get_software_root('OpenBLAS') + if flexiblas: + cmake_options = ' '.join([cmake_options, '-DBLA_VENDOR=FlexiBLAS']) + elif mkl: + cmake_options = ' '.join([cmake_options, '-DBLA_VENDOR=Intel']) + elif openblas: + cmake_options = ' '.join([cmake_options, '-DBLA_VENDOR=OpenBLAS']) + else: + raise EasyBuildError("No FlexiBLAS/MKL/OpenBLAS is founded. Please assign BLA_VENDOR in cmake_options in easyconfigs") + cmake_options = ' '.join([cmake_options, self.cfg['cmake_options']]) + self.cfg.update('prebuildopts', 'CMAKE_OPTIONS="%s"' % cmake_options) def install_step(self): """Install by copying the contents of the builddir to the installdir (preserving permissions)""" From 85682f4c73c75497c3fdc5dc72093cfe01d3ff10 Mon Sep 17 00:00:00 2001 From: Chia-Jung Hsu Date: Tue, 14 Nov 2023 17:33:49 +0100 Subject: [PATCH 053/145] Break long line --- easybuild/easyblocks/s/suitesparse.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/s/suitesparse.py b/easybuild/easyblocks/s/suitesparse.py index 6c9d27a235..029b2f1276 100644 --- a/easybuild/easyblocks/s/suitesparse.py +++ b/easybuild/easyblocks/s/suitesparse.py @@ -191,6 +191,7 @@ def configure_step(self): flexiblas = get_software_root('FlexiBLAS') mkl = get_software_root('imkl') openblas = get_software_root('OpenBLAS') + if flexiblas: cmake_options = ' '.join([cmake_options, '-DBLA_VENDOR=FlexiBLAS']) elif mkl: @@ -198,7 +199,8 @@ def configure_step(self): elif openblas: cmake_options = ' '.join([cmake_options, '-DBLA_VENDOR=OpenBLAS']) else: - raise EasyBuildError("No FlexiBLAS/MKL/OpenBLAS is founded. Please assign BLA_VENDOR in cmake_options in easyconfigs") + raise EasyBuildError("No FlexiBLAS/MKL/OpenBLAS is founded. " + "Please assign BLA_VENDOR in cmake_options in easyconfigs") cmake_options = ' '.join([cmake_options, self.cfg['cmake_options']]) self.cfg.update('prebuildopts', 'CMAKE_OPTIONS="%s"' % cmake_options) From 7739119f1641d8ba8c1d0c17bf797239cfafeaa2 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 15 Nov 2023 13:57:16 +0100 Subject: [PATCH 054/145] update error detection for PyTorch tests --- easybuild/easyblocks/p/pytorch.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/p/pytorch.py b/easybuild/easyblocks/p/pytorch.py index 58a5e9bdb8..f40cf23f92 100644 --- a/easybuild/easyblocks/p/pytorch.py +++ b/easybuild/easyblocks/p/pytorch.py @@ -288,7 +288,9 @@ def test_step(self): failed_test_cases = re.findall(regex, tests_out, re.M) # And patterns like: # FAILED test_ops_gradients.py::TestGradientsCPU::test_fn_grad_linalg_det_singular_cpu_complex128 - [snip] - regex = r"^(FAILED) \w+\.py.*::(test_.*?) - " + # FAILED [22.8699s] test_sparse_csr.py::TestSparseCompressedCPU::test_invalid_input_csr_large_cpu - [snip] + # FAILED [0.0623s] dynamo/test_dynamic_shapes.py::DynamicShapesExportTests::test_predispatch - [snip] + regex = r"^(FAILED) (?:\[.*?\] )?(?:\w|/)+\.py.*::(test_.*?) - " failed_test_cases.extend(re.findall(regex, tests_out, re.M)) if failed_test_cases: errored_test_cases = sorted(m[1] for m in failed_test_cases if m[0] == 'ERROR') @@ -350,11 +352,13 @@ def get_count_for_pattern(regex, text): # OR: # ===================== 2 failed, 128 passed, 2 skipped, 2 warnings in 63.43s (01:03:43) ========= # FINISHED PRINTING LOG FILE + # # test_quantization failed! regex = ( r"^=+ (?P.*) in [0-9]+\.*[0-9]*[a-zA-Z]* (\([0-9]+:[0-9]+:[0-9]+\) )?=+$\n" - r"(?:^(?:(?!failed!).)*$\n){0,5}" + r"(?:.*FINISHED PRINTING LOG FILE.*\n)?" + r"(?:^\s*\n)*" r"(?P.*) failed!$" ) @@ -414,7 +418,23 @@ def get_count_for_pattern(regex, text): # Calculate total number of unsuccesful and total tests failed_test_cnt = failure_cnt + error_cnt - test_cnt = sum(int(hit) for hit in re.findall(r"^Ran (?P[0-9]+) tests in", tests_out, re.M)) + # Pattern for tests ran with unittest like: + # Ran 3 tests in 0.387s + regex = r"^Ran (?P[0-9]+) tests in" + test_cnt = sum(int(hit) for hit in re.findall(regex, tests_out, re.M)) + # Pattern for tests ran with pytest like: + # ============ 286 passed, 18 skipped, 2 xfailed in 38.71s ============ + regex = r"=+ (?P.*) in \d+.* =+\n" + count_patterns = [re.compile(r"([0-9]+) " + reason) for reason in [ + "failed", + "passed", + "skipped", + "deselected", + "xfailed", + "xpassed", + ]] + for m in re.finditer(regex, tests_out, re.M): + test_cnt += sum(get_count_for_pattern(p, m.group("summary")) for p in count_patterns) if failed_test_cnt > 0: max_failed_tests = self.cfg['max_failed_tests'] From 0fa67574c55b0ba8cc5c6bfdf2e8492f6cc9f603 Mon Sep 17 00:00:00 2001 From: "A.H. Kole" Date: Wed, 15 Nov 2023 15:16:26 +0100 Subject: [PATCH 055/145] Added easyblock for flook that includes install step --- easybuild/easyblocks/f/flook.py | 43 +++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 easybuild/easyblocks/f/flook.py diff --git a/easybuild/easyblocks/f/flook.py b/easybuild/easyblocks/f/flook.py new file mode 100644 index 0000000000..5cd4ced1a8 --- /dev/null +++ b/easybuild/easyblocks/f/flook.py @@ -0,0 +1,43 @@ +## +# Copyright 2023 Utrecht University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for building and installing flook, implemented as an easyblock + +@author: Arnold Kole (Utrecht University) +""" + +from easybuild.easyblocks.generic.configuremake import ConfigureMake + + +class EB_flook(ConfigureMake): + """Support for building/installing flook.""" + + def configure_step(self): + # flook has no configure step + pass + + def install_step(self): + self.cfg.update('install_cmd', 'PREFIX=%s' % self.installdir) + super(EB_flook, self).install_step() From ca27be05cfdde3d1f6e414428866852beb0f33ee Mon Sep 17 00:00:00 2001 From: "A.H. Kole" Date: Wed, 15 Nov 2023 16:04:44 +0100 Subject: [PATCH 056/145] Moved sanity check and buildopts into easyblock --- easybuild/easyblocks/f/flook.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/easybuild/easyblocks/f/flook.py b/easybuild/easyblocks/f/flook.py index 5cd4ced1a8..2e289de436 100644 --- a/easybuild/easyblocks/f/flook.py +++ b/easybuild/easyblocks/f/flook.py @@ -34,6 +34,15 @@ class EB_flook(ConfigureMake): """Support for building/installing flook.""" + def __init__(self, *args, **kwargs): + # call out to original constructor first, so 'self' (i.e. the class instance) is initialised + super(EB_flook, self).__init__(*args, **kwargs) + + # Set some default options + local_comp_flags = 'VENDOR="gnu" FFLAGS="$FFLAGS" CFLAGS="$CFLAGS"' + self.cfg.update('buildopts', 'liball %s' % local_comp_flags) + self.cfg.update('parallel', 1) + def configure_step(self): # flook has no configure step pass @@ -41,3 +50,12 @@ def configure_step(self): def install_step(self): self.cfg.update('install_cmd', 'PREFIX=%s' % self.installdir) super(EB_flook, self).install_step() + + def sanity_check_step(self): + custom_paths = { + 'files': ['include/flook.mod', 'lib/libflook.a', 'lib/libflookall.a', 'lib/pkgconfig/flook.pc'], + 'dirs': [], + } + + # call out to parent to do the actual sanity checking, pass through custom paths + super(EB_flook, self).sanity_check_step(custom_paths=custom_paths) From ae9b82cda830e77b45651f4a8a11b22ebf051d63 Mon Sep 17 00:00:00 2001 From: "A.H. Kole" Date: Wed, 15 Nov 2023 16:07:26 +0100 Subject: [PATCH 057/145] Fix setting of parallel in constructor --- easybuild/easyblocks/f/flook.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/f/flook.py b/easybuild/easyblocks/f/flook.py index 2e289de436..3aab13f72f 100644 --- a/easybuild/easyblocks/f/flook.py +++ b/easybuild/easyblocks/f/flook.py @@ -41,7 +41,7 @@ def __init__(self, *args, **kwargs): # Set some default options local_comp_flags = 'VENDOR="gnu" FFLAGS="$FFLAGS" CFLAGS="$CFLAGS"' self.cfg.update('buildopts', 'liball %s' % local_comp_flags) - self.cfg.update('parallel', 1) + self.cfg['parallel'] = 1 def configure_step(self): # flook has no configure step From a60f4a26175a315b74eab6a49d440f3462ffd0fa Mon Sep 17 00:00:00 2001 From: "A.H. Kole" Date: Wed, 15 Nov 2023 17:24:33 +0100 Subject: [PATCH 058/145] Automatically detect correct vendor based on toolchain --- easybuild/easyblocks/f/flook.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/f/flook.py b/easybuild/easyblocks/f/flook.py index 3aab13f72f..c2a3ebd6ad 100644 --- a/easybuild/easyblocks/f/flook.py +++ b/easybuild/easyblocks/f/flook.py @@ -38,8 +38,22 @@ def __init__(self, *args, **kwargs): # call out to original constructor first, so 'self' (i.e. the class instance) is initialised super(EB_flook, self).__init__(*args, **kwargs) - # Set some default options - local_comp_flags = 'VENDOR="gnu" FFLAGS="$FFLAGS" CFLAGS="$CFLAGS"' + # Determine vendor + vendor = None + if self.toolchain.COMPILER_FAMILY == 'Clang': + vendor = 'clang' + elif self.toolchain.COMPILER_FAMILY == 'GCC': + vendor = 'gnu' + elif self.toolchain.COMPILER_FAMILY == 'Intel': + vendor = 'intel' + elif self.toolchain.COMPILER_FAMILY == 'PGI': + vendor = 'pgi' + + # Set some default options + if vendor is not None: + local_comp_flags = 'VENDOR="%s" FFLAGS="$FFLAGS" CFLAGS="$CFLAGS"' % vendor + else: + local_comp_flags = 'FFLAGS="$FFLAGS" CFLAGS="$CFLAGS"' self.cfg.update('buildopts', 'liball %s' % local_comp_flags) self.cfg['parallel'] = 1 From 01ccf505bee25529a2a661df2cc87581ff691b70 Mon Sep 17 00:00:00 2001 From: "A.H. Kole" Date: Wed, 15 Nov 2023 17:28:25 +0100 Subject: [PATCH 059/145] Convert tabs to spaces for indentation --- easybuild/easyblocks/f/flook.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/f/flook.py b/easybuild/easyblocks/f/flook.py index c2a3ebd6ad..635fa99854 100644 --- a/easybuild/easyblocks/f/flook.py +++ b/easybuild/easyblocks/f/flook.py @@ -38,7 +38,7 @@ def __init__(self, *args, **kwargs): # call out to original constructor first, so 'self' (i.e. the class instance) is initialised super(EB_flook, self).__init__(*args, **kwargs) - # Determine vendor + # Determine vendor vendor = None if self.toolchain.COMPILER_FAMILY == 'Clang': vendor = 'clang' @@ -49,7 +49,7 @@ def __init__(self, *args, **kwargs): elif self.toolchain.COMPILER_FAMILY == 'PGI': vendor = 'pgi' - # Set some default options + # Set some default options if vendor is not None: local_comp_flags = 'VENDOR="%s" FFLAGS="$FFLAGS" CFLAGS="$CFLAGS"' % vendor else: From d38efa954197040377c8158dd1ae444486e9a87a Mon Sep 17 00:00:00 2001 From: Leonard Nielsen Date: Thu, 16 Nov 2023 15:34:04 +0100 Subject: [PATCH 060/145] Updated WIEN2k easyblock for 23.2 --- easybuild/easyblocks/w/wien2k.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/easybuild/easyblocks/w/wien2k.py b/easybuild/easyblocks/w/wien2k.py index 07c32ad40c..acf69b225b 100644 --- a/easybuild/easyblocks/w/wien2k.py +++ b/easybuild/easyblocks/w/wien2k.py @@ -248,6 +248,9 @@ def configure_step(self): libxcquestion4 = 'Please enter the lib-directory of your LIBXC-installation (usually lib or lib64)!:' libxcquestion5 = 'LIBXC (usually not needed, ONLY for experts who want to play with different DFT options. ' libxcquestion5 += 'It must have been installed before)? (y,N):' + libxcquestion6 = 'Would you like to use LIBXC (needed ONLY for self-consistent gKS mGGA calculations, ' + libxcquestion6 += 'for the stress tensor and experts who want to play with different DFT options. ' + libxcquestion6 += 'It must have been installed before)? (y,N):' if libxcroot: qanda.update({ @@ -256,11 +259,13 @@ def configure_step(self): libxcquestion3: libxcroot, libxcquestion4: 'lib', libxcquestion5: 'y', + libxcquestion6: 'y', }) else: qanda.update({ libxcquestion1: 'N', libxcquestion5: 'N', + libxcquestion6: 'N', }) if LooseVersion(self.version) >= LooseVersion("17"): From be600d55466f9b52657e6497594a4cb2c4d176ac Mon Sep 17 00:00:00 2001 From: Chia-Jung Hsu Date: Wed, 22 Nov 2023 18:30:34 +0100 Subject: [PATCH 061/145] Fix BLAS dependency check --- easybuild/easyblocks/s/suitesparse.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/easybuild/easyblocks/s/suitesparse.py b/easybuild/easyblocks/s/suitesparse.py index 029b2f1276..1276b12203 100644 --- a/easybuild/easyblocks/s/suitesparse.py +++ b/easybuild/easyblocks/s/suitesparse.py @@ -40,6 +40,7 @@ from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.framework.easyconfig import CUSTOM +from easybuild.tools import toolchain from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import mkdir, write_file from easybuild.tools.modules import get_software_root @@ -188,20 +189,20 @@ def configure_step(self): # CMAKE_INSTALL_PREFIX is managed by easybuild cmake_options = '-DCMAKE_INSTALL_PREFIX=%s' % self.installdir - flexiblas = get_software_root('FlexiBLAS') - mkl = get_software_root('imkl') - openblas = get_software_root('OpenBLAS') - - if flexiblas: - cmake_options = ' '.join([cmake_options, '-DBLA_VENDOR=FlexiBLAS']) - elif mkl: - cmake_options = ' '.join([cmake_options, '-DBLA_VENDOR=Intel']) - elif openblas: - cmake_options = ' '.join([cmake_options, '-DBLA_VENDOR=OpenBLAS']) + lapack_lib = self.toolchain.lapack_family() + if '-DBLA_VENDOR=' in self.cfg['cmake_options']: + blas_lapack = '' + elif lapack_lib == toolchain.FLEXIBLAS: + blas_lapack = '-DBLA_VENDOR=FlexiBLAS' + elif lapack_lib == toolchain.INTELMKL: + blas_lapack = '-DBLA_VENDOR=Intel' + elif lapack_lib == toolchain.OPENBLAS: + blas_lapack = '-DBLA_VENDOR=OpenBLAS' else: - raise EasyBuildError("No FlexiBLAS/MKL/OpenBLAS is founded. " + raise EasyBuildError("BLA_VENDOR is not assigned and FlexiBLAS/MKL/OpenBLAS are not found. " "Please assign BLA_VENDOR in cmake_options in easyconfigs") - cmake_options = ' '.join([cmake_options, self.cfg['cmake_options']]) + + cmake_options = ' '.join([cmake_options, blas_lapack, self.cfg['cmake_options']]) self.cfg.update('prebuildopts', 'CMAKE_OPTIONS="%s"' % cmake_options) def install_step(self): From 82d006b12b64114c4f87e24cba9f8bc86fe9172b Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 22 Nov 2023 19:17:37 +0000 Subject: [PATCH 062/145] Adapt intel_compilers easyblock to version 2024 Involves the following changes: * move "latest" symlink resolution specific to tbb to intelbase, so it can apply generically * use tbb_subdir and compilers_subdir properties for compatibility * compilers_subdir dropped "/linux" * icc and icpc were removed, ifort moved to same directory as icx etc. * lib directories were merged * man directory was moved --- easybuild/easyblocks/generic/intelbase.py | 16 ++++++++++ easybuild/easyblocks/i/intel_compilers.py | 36 ++++++++++++++--------- 2 files changed, 38 insertions(+), 14 deletions(-) diff --git a/easybuild/easyblocks/generic/intelbase.py b/easybuild/easyblocks/generic/intelbase.py index cab94b770c..142f69cfd8 100644 --- a/easybuild/easyblocks/generic/intelbase.py +++ b/easybuild/easyblocks/generic/intelbase.py @@ -111,6 +111,22 @@ def __init__(self, *args, **kwargs): self.home_subdir_local = os.path.join(common_tmp_dir, os.environ.get('USER', 'nouser'), 'easybuild_intel') self.install_components = None + # dictionary to keep track of "latest" directory symlinks + # the target may only have major.minor, and tbb may have a lower version number than the compiler + # for example compiler 2021.1.2 has tbb 2021.1.1, 2024.0.0 has directory name 2024.0 + self._latest_subdir = {} + + def get_versioned_subdir(self, subdir): + """Return versioned directory that the 'latest' symlink points to in subdir""" + if subdir not in self._latest_subdir: + if os.path.islink(os.path.join(self.installdir, subdir, 'latest')): + version = os.readlink(os.path.join(self.installdir, subdir, 'latest')) + else: + version = 'latest' + latest_subdir = os.path.join(subdir, version) + self._latest_subdir[subdir] = latest_subdir + self.log.debug('Determined versioned directory for %s: %s', subdir, version) + return self._latest_subdir[subdir] def get_guesses_tools(self): """Find reasonable paths for a subset of Intel tools, ignoring CPATH, LD_LIBRARY_PATH and LIBRARY_PATH""" diff --git a/easybuild/easyblocks/i/intel_compilers.py b/easybuild/easyblocks/i/intel_compilers.py index 3d6d9b030e..081cdeb1b8 100644 --- a/easybuild/easyblocks/i/intel_compilers.py +++ b/easybuild/easyblocks/i/intel_compilers.py @@ -51,10 +51,16 @@ def __init__(self, *args, **kwargs): if LooseVersion(self.version) < LooseVersion('2021'): raise EasyBuildError("Invalid version %s, should be >= 2021.x" % self.version) - self.compilers_subdir = os.path.join('compiler', self.version, 'linux') - # note that tbb may have a lower version number than the compiler, so use 'latest' symlink - # for example compiler 2021.1.2 has tbb 2021.1.1. - self.tbb_subdir = os.path.join('tbb', 'latest') + @property + def compilers_subdir(self): + compilers_subdir = self.get_versioned_subdir('compiler') + if LooseVersion(self.version) < LooseVersion('2024'): + compilers_subdir = os.path.join(latest_subdir, 'linux') + return compilers_subdir + + @property + def tbb_subdir(self): + return self.get_versioned_subdir('tbb') def prepare_step(self, *args, **kwargs): """ @@ -92,7 +98,6 @@ def sanity_check_step(self): Custom sanity check for Intel compilers. """ - classic_compiler_cmds = ['icc', 'icpc', 'ifort'] oneapi_compiler_cmds = [ 'dpcpp', # Intel oneAPI Data Parallel C++ compiler 'icx', # oneAPI Intel C compiler @@ -100,8 +105,14 @@ def sanity_check_step(self): 'ifx', # oneAPI Intel Fortran compiler ] bindir = os.path.join(self.compilers_subdir, 'bin') - classic_compiler_paths = [os.path.join(bindir, x) for x in oneapi_compiler_cmds] - oneapi_compiler_paths = [os.path.join(bindir, 'intel64', x) for x in classic_compiler_cmds] + oneapi_compiler_paths = [os.path.join(bindir, x) for x in oneapi_compiler_cmds] + if LooseVersion(self.version) >= LooseVersion('2024'): + classic_compiler_cmds = ['ifort'] + classic_bindir = bindir + else: + classic_compiler_cmds = ['icc', 'icpc', 'ifort'] + classic_bindir = os.path.join(bindir, 'intel64') + classic_compiler_paths = [os.path.join(classic_bindir, x) for x in classic_compiler_cmds] custom_paths = { 'files': classic_compiler_paths + oneapi_compiler_paths, @@ -131,12 +142,7 @@ def make_module_req_guess(self): os.path.join('compiler', 'lib', 'intel64_lin'), ] libdirs = [os.path.join(self.compilers_subdir, x) for x in libdirs] - # resolve 'latest' symlink for tbb (if module guess is run with install in place) - if os.path.islink(os.path.join(self.installdir, self.tbb_subdir)): - tbb_version = os.readlink(os.path.join(self.installdir, self.tbb_subdir)) - else: - tbb_version = 'latest' - tbb_subdir = os.path.join('tbb', tbb_version) + tbb_subdir = self.tbb_subdir tbb_libsubdir = os.path.join(tbb_subdir, 'lib', 'intel64') libdirs.append(os.path.join(tbb_libsubdir, get_tbb_gccprefix(os.path.join(self.installdir, tbb_libsubdir)))) @@ -148,10 +154,12 @@ def make_module_req_guess(self): 'LD_LIBRARY_PATH': libdirs, 'LIBRARY_PATH': libdirs, 'MANPATH': [ - os.path.join('compiler', self.version, 'documentation', 'en', 'man', 'common'), + os.path.join(os.path.dirname(self.compilers_subdir), 'documentation', 'en', 'man', 'common'), + os.path.join(self.compilers_subdir, 'share', 'man'), ], 'OCL_ICD_FILENAMES': [ os.path.join(self.compilers_subdir, 'lib', 'x64', 'libintelocl.so'), + os.path.join(self.compilers_subdir, 'lib', 'libintelocl.so'), ], 'CPATH': [ os.path.join(tbb_subdir, 'include'), From 222053fbbe28b4bbecf92bfb1a7fb60cd0b89f3d Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 22 Nov 2023 19:24:32 +0000 Subject: [PATCH 063/145] resolve undefined name caught by the hound --- easybuild/easyblocks/i/intel_compilers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/i/intel_compilers.py b/easybuild/easyblocks/i/intel_compilers.py index 081cdeb1b8..4ed76e2f25 100644 --- a/easybuild/easyblocks/i/intel_compilers.py +++ b/easybuild/easyblocks/i/intel_compilers.py @@ -55,7 +55,7 @@ def __init__(self, *args, **kwargs): def compilers_subdir(self): compilers_subdir = self.get_versioned_subdir('compiler') if LooseVersion(self.version) < LooseVersion('2024'): - compilers_subdir = os.path.join(latest_subdir, 'linux') + compilers_subdir = os.path.join(compilers_subdir, 'linux') return compilers_subdir @property From 7b30c215171f54348840ff3bdb7eadf93c81aba8 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Thu, 23 Nov 2023 11:58:29 +0100 Subject: [PATCH 064/145] disable LLVM build downloads from CI in Rust --- easybuild/easyblocks/r/rust.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easybuild/easyblocks/r/rust.py b/easybuild/easyblocks/r/rust.py index ee66f6854e..777d962391 100644 --- a/easybuild/easyblocks/r/rust.py +++ b/easybuild/easyblocks/r/rust.py @@ -59,6 +59,9 @@ def configure_step(self): self.cfg.update('configopts', "--sysconfdir=%s" % os.path.join(self.installdir, 'etc')) + # old llvm builds from CI get deleted after a certain time + self.cfg.update('configopts', "--set=llvm.download-ci-llvm=false") + # don't use Ninja if it is not listed as a build dependency; # may be because Ninja requires Python, and Rust is a build dependency for cryptography # which may be included as an extension with Python From 0e681978b038fae48d9f085bedc0698200bab57c Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Fri, 24 Nov 2023 12:30:58 +0000 Subject: [PATCH 065/145] Intel MPI easyblock adjustments for v2021.11 * use get_versioned_subdir() * Fortran .mod files moved from include to include/mpi * test.c moved from test to opt/mpi/test * use mpicc -cc=$CC to pickup present compiler (could be icx) * man pages moved from man to share/man * adjust aliases to not use icc/icpc for 2024 compilers and add aliases for icx/icpx/ifx. --- easybuild/easyblocks/i/impi.py | 45 ++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/easybuild/easyblocks/i/impi.py b/easybuild/easyblocks/i/impi.py index 7eee0df099..497b0659b1 100644 --- a/easybuild/easyblocks/i/impi.py +++ b/easybuild/easyblocks/i/impi.py @@ -208,7 +208,7 @@ def sanity_check_step(self): mpi_mods.extend(['mpi_base.mod', 'mpi_constants.mod', 'mpi_sizeofs.mod']) if impi_ver >= LooseVersion('2021'): - mpi_subdir = os.path.join('mpi', self.version) + mpi_subdir = self.get_versioned_subdir('mpi') bin_dir = os.path.join(mpi_subdir, 'bin') include_dir = os.path.join(mpi_subdir, 'include') lib_dir = os.path.join(mpi_subdir, 'lib', 'release') @@ -223,11 +223,15 @@ def sanity_check_step(self): lib_dir = 'lib%s' % suff mpi_mods.extend(['i_malloc.h']) + mpi_mods_dir = include_dir + if impi_ver >= LooseVersion('2021.11'): + mpi_mods_dir = os.path.join(mpi_mods_dir, 'mpi') + shlib_ext = get_shared_lib_ext() custom_paths = { 'files': [os.path.join(bin_dir, 'mpi%s' % x) for x in ['icc', 'icpc', 'ifort']] + [os.path.join(include_dir, 'mpi%s.h' % x) for x in ['cxx', 'f', '', 'o', 'of']] + - [os.path.join(include_dir, x) for x in mpi_mods] + + [os.path.join(mpi_mods_dir, x) for x in mpi_mods] + [os.path.join(lib_dir, 'libmpi.%s' % shlib_ext)] + [os.path.join(lib_dir, 'libmpi.a')], 'dirs': [], @@ -239,7 +243,10 @@ def sanity_check_step(self): if impi_ver >= LooseVersion('2017'): # Add minimal test program to sanity checks if impi_ver >= LooseVersion('2021'): - impi_testsrc = os.path.join(self.installdir, 'mpi', self.version, 'test', 'test.c') + impi_testsrc = os.path.join(self.installdir, self.get_versioned_subdir('mpi')) + if impi_ver >= LooseVersion('2021.11'): + impi_testsrc = os.path.join(impi_testsrc, 'opt', 'mpi') + impi_testsrc = os.path.join(impi_testsrc, 'test', 'test.c') else: impi_testsrc = os.path.join(self.installdir, 'test', 'test.c') @@ -247,12 +254,7 @@ def sanity_check_step(self): self.log.info("Adding minimal MPI test program to sanity checks: %s", impi_testsrc) # Build test program with appropriate compiler from current toolchain - comp_fam = self.toolchain.comp_family() - if comp_fam == toolchain.INTELCOMP: - build_comp = 'mpiicc' - else: - build_comp = 'mpicc' - build_cmd = "%s %s -o %s" % (build_comp, impi_testsrc, impi_testexe) + build_cmd = "mpicc -cc=%s %s -o %s" % (os.getenv('CC'), impi_testsrc, impi_testexe) # Execute test program with appropriate MPI executable for target toolchain params = {'nr_ranks': self.cfg['parallel'], 'cmd': impi_testexe} @@ -283,7 +285,7 @@ def make_module_req_guess(self): impi_ver = LooseVersion(self.version) if impi_ver >= LooseVersion('2021'): - mpi_subdir = os.path.join('mpi', self.version) + mpi_subdir = self.get_versioned_subdir('mpi') lib_dirs = [ os.path.join(mpi_subdir, 'lib'), os.path.join(mpi_subdir, 'lib', 'release'), @@ -294,10 +296,13 @@ def make_module_req_guess(self): os.path.join(mpi_subdir, 'bin'), os.path.join(mpi_subdir, 'libfabric', 'bin'), ] - manpath = os.path.join(mpi_subdir, 'man') + if impi_ver >= LooseVersion('2021.11'): + manpath = os.path.join(mpi_subdir, 'share', 'man') + else: + manpath = os.path.join(mpi_subdir, 'man') if self.cfg['ofi_internal']: - libfabric_dir = os.path.join('mpi', self.version, 'libfabric') + libfabric_dir = os.path.join(mpi_subdir, 'libfabric') lib_dirs.append(os.path.join(libfabric_dir, 'lib')) path_dirs.append(os.path.join(libfabric_dir, 'bin')) guesses['FI_PROVIDER_PATH'] = [os.path.join(libfabric_dir, 'lib', 'prov')] @@ -332,7 +337,7 @@ def make_module_extra(self, *args, **kwargs): """Overwritten from Application to add extra txt""" if LooseVersion(self.version) >= LooseVersion('2021'): - mpiroot = os.path.join(self.installdir, 'mpi', self.version) + mpiroot = os.path.join(self.installdir, self.get_versioned_subdir('mpi')) else: mpiroot = self.installdir @@ -361,11 +366,21 @@ def make_module_extra(self, *args, **kwargs): if self.cfg['set_mpi_wrapper_aliases_intel'] or self.cfg['set_mpi_wrappers_all']: # do the same for mpiicc/mpiipc/mpiifort to be consistent, even if they may not exist - txt += self.module_generator.set_alias('mpiicc', 'mpiicc -cc=icc') - txt += self.module_generator.set_alias('mpiicpc', 'mpiicpc -cxx=icpc') + if (get_software_root('intel-compilers') and + LooseVersion(get_software_version('intel-compilers')) >= LooseVersion('2024')): + txt += self.module_generator.set_alias('mpiicc', 'mpiicc -cc=icx') + txt += self.module_generator.set_alias('mpiicpc', 'mpiicpc -cxx=icpx') + else: + txt += self.module_generator.set_alias('mpiicc', 'mpiicc -cc=icc') + txt += self.module_generator.set_alias('mpiicpc', 'mpiicpc -cxx=icpc') # -fc also works, but -f90 takes precedence txt += self.module_generator.set_alias('mpiifort', 'mpiifort -f90=ifort') + if LooseVersion(self.version) >= LooseVersion('2021.11'): + txt += self.module_generator.set_alias('mpiicx', 'mpiicx -cc=icx') + txt += self.module_generator.set_alias('mpiicpx', 'mpiicpx -cxx=icpx') + txt += self.module_generator.set_alias('mpiifx', 'mpiifx -f90=ifx') + # set environment variable UCX_TLS to 'all', this works in all hardware configurations # needed with UCX regardless of the transports available (even without a Mellanox HCA) # more information in easybuilders/easybuild-easyblocks#2253 From 4273e31e8b232515d02093ce51c06af32a1894b6 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Fri, 24 Nov 2023 12:37:21 +0000 Subject: [PATCH 066/145] import get_software_version for aliases --- easybuild/easyblocks/i/impi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/i/impi.py b/easybuild/easyblocks/i/impi.py index 497b0659b1..bcded116b4 100644 --- a/easybuild/easyblocks/i/impi.py +++ b/easybuild/easyblocks/i/impi.py @@ -42,7 +42,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option from easybuild.tools.filetools import apply_regex_substitutions, change_dir, extract_file, mkdir, write_file -from easybuild.tools.modules import get_software_root +from easybuild.tools.modules import get_software_root, get_software_version from easybuild.tools.run import run_cmd from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.toolchain.mpi import get_mpi_cmd_template From 3beee09625bb37ca4eb637edab713ed336b22af9 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Fri, 24 Nov 2023 12:45:25 +0000 Subject: [PATCH 067/145] Appease Hound with indentation --- easybuild/easyblocks/i/impi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/i/impi.py b/easybuild/easyblocks/i/impi.py index bcded116b4..46bb3fcad1 100644 --- a/easybuild/easyblocks/i/impi.py +++ b/easybuild/easyblocks/i/impi.py @@ -367,7 +367,7 @@ def make_module_extra(self, *args, **kwargs): if self.cfg['set_mpi_wrapper_aliases_intel'] or self.cfg['set_mpi_wrappers_all']: # do the same for mpiicc/mpiipc/mpiifort to be consistent, even if they may not exist if (get_software_root('intel-compilers') and - LooseVersion(get_software_version('intel-compilers')) >= LooseVersion('2024')): + LooseVersion(get_software_version('intel-compilers')) >= LooseVersion('2024')): txt += self.module_generator.set_alias('mpiicc', 'mpiicc -cc=icx') txt += self.module_generator.set_alias('mpiicpc', 'mpiicpc -cxx=icpx') else: From f9b544a9de3f67e85b3a7f4d43868a1123390dd3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 24 Nov 2023 20:36:24 +0100 Subject: [PATCH 068/145] update numpy easyblock for v1.26+ --- easybuild/easyblocks/n/numpy.py | 50 ++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/n/numpy.py b/easybuild/easyblocks/n/numpy.py index f42b914e55..734953382c 100644 --- a/easybuild/easyblocks/n/numpy.py +++ b/easybuild/easyblocks/n/numpy.py @@ -217,6 +217,31 @@ def get_libs_for_mkl(varname): cmd = "%s setup.py config" % self.python_cmd run_cmd(cmd, log_all=True, simple=True) + if LooseVersion(self.version) >= LooseVersion('1.26'): + # control BLAS/LAPACK library being used + # see https://github.com/numpy/numpy/blob/v1.26.2/doc/source/release/1.26.1-notes.rst#build-system-changes + # and 'blas-order' in https://github.com/numpy/numpy/blob/v1.26.2/meson_options.txt + blas_lapack_names = { + toolchain.BLIS: 'blis', + toolchain.FLEXIBLAS: 'flexiblas', + toolchain.LAPACK: 'lapack', + toolchain.INTELMKL: 'mkl', + toolchain.OPENBLAS: 'openblas', + } + blas_family = self.toolchain.blas_family() + if blas_family in blas_lapack_names: + self.cfg.update('installopts', "-Csetup-args=-Dblas=" + blas_lapack_names[blas_family]) + else: + raise EasyBuildError("Unknown BLAS library for numpy %s: %s", self.version, blas_family) + + lapack_family = self.toolchain.lapack_family() + if lapack_family in blas_lapack_names: + self.cfg.update('installopts', "-Csetup-args=-Dlapack=" + blas_lapack_names[lapack_family]) + else: + raise EasyBuildError("Unknown LAPACK library for numpy %s: %s", self.version, lapack_family) + + self.cfg.update('installopts', "-Csetup-args=-Dallow-noblas=false") + def test_step(self): """Run available numpy unit tests, and more.""" @@ -332,7 +357,30 @@ def sanity_check_step(self, *args, **kwargs): custom_commands = [] - if LooseVersion(self.version) >= LooseVersion("1.10"): + if LooseVersion(self.version) >= LooseVersion('1.26'): + # make sure BLAS library was found + blas_check_pytxt = '; '.join([ + "import numpy", + "numpy_config = numpy.show_config(mode='dicts')", + "numpy_build_deps = numpy_config['Build Dependencies']", + "blas_found = numpy_build_deps['blas']['found']", + "assert blas_found", + ]) + custom_commands.append('python -c "%s"' % blas_check_pytxt) + + # if FlexiBLAS is used, make sure we are linking to it + # (rather than directly to a backend library like OpenBLAS or Intel MKL) + if self.toolchain.blas_family() == toolchain.FLEXIBLAS: + blas_check_pytxt = '; '.join([ + "import numpy", + "numpy_config = numpy.show_config(mode='dicts')", + "numpy_build_deps = numpy_config['Build Dependencies']", + "blas_name = numpy_build_deps['blas']['name']", + "assert blas_name == 'flexiblas', 'BLAS library should be flexiblas, found %s' % blas_name", + ]) + custom_commands.append('python -c "%s"' % blas_check_pytxt) + + elif LooseVersion(self.version) >= LooseVersion('1.10'): # generic check to see whether numpy v1.10.x and up was built against a CBLAS-enabled library # cfr. https://github.com/numpy/numpy/issues/6675#issuecomment-162601149 blas_check_pytxt = '; '.join([ From 7d67ee996aa30bc9f61571e1b143645fda3478f6 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Mon, 27 Nov 2023 12:15:56 +0000 Subject: [PATCH 069/145] Adjustments for imkl and imkl-FFTW 2024 * determine mkl_basedir like for intel-compilers via latest symlink * interfaces -> share/mkl/interfaces * tools/builder -> share/mkl/tools/builder * linux/compiler/lib/intel64_lin -> lib * examples -> share/doc/mkl/examples --- easybuild/easyblocks/generic/intelbase.py | 3 ++ easybuild/easyblocks/i/imkl.py | 50 ++++++++++++++--------- 2 files changed, 34 insertions(+), 19 deletions(-) diff --git a/easybuild/easyblocks/generic/intelbase.py b/easybuild/easyblocks/generic/intelbase.py index 142f69cfd8..31f44d1935 100644 --- a/easybuild/easyblocks/generic/intelbase.py +++ b/easybuild/easyblocks/generic/intelbase.py @@ -128,6 +128,9 @@ def get_versioned_subdir(self, subdir): self.log.debug('Determined versioned directory for %s: %s', subdir, version) return self._latest_subdir[subdir] + def set_versioned_subdir(self, subdir, path): + self._latest_subdir[subdir] = path + def get_guesses_tools(self): """Find reasonable paths for a subset of Intel tools, ignoring CPATH, LD_LIBRARY_PATH and LIBRARY_PATH""" diff --git a/easybuild/easyblocks/i/imkl.py b/easybuild/easyblocks/i/imkl.py index a342a4686c..883b773535 100644 --- a/easybuild/easyblocks/i/imkl.py +++ b/easybuild/easyblocks/i/imkl.py @@ -79,14 +79,26 @@ def __init__(self, *args, **kwargs): self.cdftlibs = [] self.mpi_spec = None + if self.cfg['flexiblas'] is None: + self.cfg['flexiblas'] = LooseVersion(self.version) >= LooseVersion('2021') + + if LooseVersion(self.version) >= LooseVersion('2024'): + self.examples_subdir = os.path.join('share', 'doc', 'mkl', 'examples') + self.compiler_libdir = 'lib' + else: + self.examples_subdir = 'examples' + self.compiler_libdir = os.path.join('linux', 'compiler', 'lib', 'intel64_lin') + + @property + def mkl_basedir(self): if LooseVersion(self.version) >= LooseVersion('2021'): - self.mkl_basedir = os.path.join('mkl', self.version) - if self.cfg['flexiblas'] is None: - self.cfg['flexiblas'] = True + return self.get_versioned_subdir('mkl') else: - self.mkl_basedir = 'mkl' - if self.cfg['flexiblas'] is None: - self.cfg['flexiblas'] = False + return 'mkl' + + @mkl_basedir.setter + def mkl_basedir(self, path): + self.set_versioned_subdir('mkl', path) def prepare_step(self, *args, **kwargs): """Prepare build environment.""" @@ -162,7 +174,10 @@ def build_mkl_fftw_interfaces(self, libdir): loosever = LooseVersion(self.version) if loosever >= LooseVersion('10.3'): - intsubdir = os.path.join(self.mkl_basedir, 'interfaces') + intsubdir = self.mkl_basedir + if loosever >= LooseVersion('2024'): + intsubdir = os.path.join(intsubdir, 'share', 'mkl') + intsubdir = os.path.join(intsubdir, 'interfaces') inttarget = 'libintel64' else: intsubdir = 'interfaces' @@ -306,8 +321,10 @@ def build_mkl_flexiblas(self, flexiblasdir): and libflexiblas_imkl_sequential.so. They can be used as FlexiBLAS backends via FLEXIBLAS_LIBRARY_PATH. """ - builderdir = os.path.join(self.installdir, self.mkl_basedir, 'tools', 'builder') - change_dir(builderdir) + builderdir = os.path.join('tools', 'builder') + if LooseVersion(self.version) >= LooseVersion('2024'): + builderdir = os.path.join('share', 'mkl', builderdir) + change_dir(os.path.join(self.installdir, self.mkl_basedir, builderdir)) mkdir(flexiblasdir, parents=True) # concatenate lists of all BLAS, CBLAS and LAPACK functions @@ -318,8 +335,7 @@ def build_mkl_flexiblas(self, flexiblasdir): with open(lst + "_example_list") as src: shutil.copyfileobj(src, dst) - compilerdir = os.path.join('..', '..', '..', '..', 'compiler', self.version, - 'linux', 'compiler', 'lib', 'intel64_lin') + compilerdir = os.path.join(self.installdir, self.get_versioned_subdir('compiler'), self.compiler_libdir) # IFACE_COMP_PART=gf gives the gfortran calling convention that FlexiBLAS expects cmds = ["make libintel64 IFACE_COMP_PART=gf export=%s name=%s" % ( listfilename, os.path.join(flexiblasdir, 'libflexiblas_imkl_')) + @@ -339,7 +355,7 @@ def post_install_step(self): super(EB_imkl, self).post_install_step() # extract examples - examples_subdir = os.path.join(self.installdir, self.mkl_basedir, 'examples') + examples_subdir = os.path.join(self.installdir, self.mkl_basedir, self.examples_subdir) if os.path.exists(examples_subdir): cwd = change_dir(examples_subdir) for examples_tarball in glob.glob('examples_*.tgz'): @@ -515,7 +531,7 @@ def make_module_req_guess(self): raise EasyBuildError("32-bit not supported yet for IMKL v%s (>= 10.3)", self.version) else: if LooseVersion(self.version) >= LooseVersion('2021'): - compiler_subdir = os.path.join('compiler', self.version, 'linux', 'compiler', 'lib', 'intel64_lin') + compiler_subdir = os.path.join(self.get_versioned_subdir('compiler'), self.compiler_libdir) pkg_config_path = [os.path.join(self.mkl_basedir, 'tools', 'pkgconfig'), os.path.join(self.mkl_basedir, 'lib', 'pkgconfig')] else: @@ -580,16 +596,12 @@ def make_module_extra(self): if 'MKL_EXAMPLES' not in self.cfg['modextravars']: self.cfg.update('modextravars', { - 'MKL_EXAMPLES': os.path.join(self.installdir, self.mkl_basedir, 'examples'), + 'MKL_EXAMPLES': os.path.join(self.installdir, self.mkl_basedir, self.examples_subdir), }) txt = super(EB_imkl, self).make_module_extra() - if LooseVersion(self.version) >= LooseVersion('2021'): - mklroot = os.path.join(self.installdir, 'mkl', self.version) - else: - mklroot = os.path.join(self.installdir, 'mkl') - + mklroot = os.path.join(self.installdir, self.mkl_basedir) txt += self.module_generator.set_environment('MKLROOT', mklroot) return txt From 4a6a1add599bac438a1bedc50427e54839c90637 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Wed, 29 Nov 2023 17:16:37 +0000 Subject: [PATCH 070/145] Use EB_COMSOL_LICENSE_FILE variable name --- easybuild/easyblocks/c/comsol.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/c/comsol.py b/easybuild/easyblocks/c/comsol.py index 38d778a623..6be902ae17 100644 --- a/easybuild/easyblocks/c/comsol.py +++ b/easybuild/easyblocks/c/comsol.py @@ -59,7 +59,7 @@ def extract_step(self): def configure_step(self): """Configure COMSOL installation: create license file.""" - default_lic_env_var = 'LMCOMSOL_LICENSE_FILE' + default_lic_env_var = 'EB_COMSOL_LICENSE_FILE' lic_specs, self.license_env_var = find_flexlm_license(custom_env_vars=[default_lic_env_var], lic_specs=[self.cfg['license_file']]) From 662fd0d52daa4d02de75fdcd905c2be6b6e58801 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Thu, 30 Nov 2023 12:04:41 +0100 Subject: [PATCH 071/145] Add requirement for EULA acceptance to CUDA easyblock --- easybuild/easyblocks/c/cuda.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/easybuild/easyblocks/c/cuda.py b/easybuild/easyblocks/c/cuda.py index 32629aab40..659e4f59dc 100644 --- a/easybuild/easyblocks/c/cuda.py +++ b/easybuild/easyblocks/c/cuda.py @@ -92,6 +92,16 @@ def __init__(self, *args, **kwargs): self.cfg.template_values['cudaarch'] = cudaarch self.cfg.generate_template_values() + def fetch_step(self, *args, **kwargs): + """Check for EULA acceptance prior to getting sources.""" + # EULA for CUDA must be accepted via --accept-eula-for EasyBuild configuration option, + # or via 'accept_eula = True' in easyconfig file + self.check_accepted_eula( + name='CUDA', + more_info='https://docs.nvidia.com/cuda/eula/index.html' + ) + return super(EB_CUDA, self).fetch_step(*args, **kwargs) + def extract_step(self): """Extract installer to have more control, e.g. options, patching Perl scripts, etc.""" execpath = self.src[0]['path'] From 714bd2cf52d16238995aa647071e48092dd97178 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 6 Dec 2023 12:58:14 +0000 Subject: [PATCH 072/145] Convert ConfigureMake easyblock to run_shell_cmd. The default for the verbose= parameter for build_step is reversed in line with the default for run_shell_cmd. --- easybuild/easyblocks/generic/configuremake.py | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/easybuild/easyblocks/generic/configuremake.py b/easybuild/easyblocks/generic/configuremake.py index 9e4066b60b..ec9f0527f3 100644 --- a/easybuild/easyblocks/generic/configuremake.py +++ b/easybuild/easyblocks/generic/configuremake.py @@ -49,7 +49,7 @@ from easybuild.tools.config import source_paths, build_option from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, adjust_permissions, compute_checksum, download_file from easybuild.tools.filetools import read_file, remove_file -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd # string that indicates that a configure script was generated by Autoconf # note: bytes string since this constant is used to check the contents of 'configure' which is read as bytes @@ -247,8 +247,8 @@ def determine_build_and_host_type(self): "EasyBuild attempts to download a recent config.guess but seems to have failed!") else: self.check_config_guess() - system_type, _ = run_cmd(self.config_guess, log_all=True) - system_type = system_type.strip() + res = run_shell_cmd(self.config_guess) + system_type = res.output.strip() self.log.info("%s returned a system type '%s'", self.config_guess, system_type) if build_type is None: @@ -323,11 +323,11 @@ def configure_step(self, cmd_prefix=''): ] + build_and_host_options + [self.cfg['configopts']] ) - (out, _) = run_cmd(cmd, log_all=True, simple=False) + res = run_shell_cmd(cmd) - return out + return res.output - def build_step(self, verbose=False, path=None): + def build_step(self, verbose=True, path=None): """ Start the actual build - typical: make -j X @@ -351,7 +351,8 @@ def build_step(self, verbose=False, path=None): ]) self.log.info("Building target '%s'", target) - (out, _) = run_cmd(cmd, path=path, log_all=True, simple=False, log_output=verbose) + res = run_shell_cmd(cmd, work_dir=path, output_file=verbose) + out = res.output return out @@ -369,9 +370,9 @@ def test_step(self): runtest = '' # Compose command filtering out empty values cmd = ' '.join([x for x in (self.cfg['pretestopts'], test_cmd, runtest, self.cfg['testopts']) if x]) - (out, _) = run_cmd(cmd, log_all=True, simple=False) + res = run_shell_cmd(cmd) - return out + return res.output def install_step(self): """ @@ -385,6 +386,6 @@ def install_step(self): self.cfg['installopts'], ]) - (out, _) = run_cmd(cmd, log_all=True, simple=False) + res = run_shell_cmd(cmd) - return out + return res.output From 1cabe3c7d792f07c75c174fd1afaafc6d1f78210 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 6 Dec 2023 13:20:46 +0000 Subject: [PATCH 073/145] Convert CMakeMake easyblock to run_shell_cmd. --- easybuild/easyblocks/generic/cmakemake.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/easybuild/easyblocks/generic/cmakemake.py b/easybuild/easyblocks/generic/cmakemake.py index 0a326a8e5a..1552b082a6 100644 --- a/easybuild/easyblocks/generic/cmakemake.py +++ b/easybuild/easyblocks/generic/cmakemake.py @@ -45,7 +45,7 @@ from easybuild.tools.filetools import change_dir, create_unused_dir, mkdir, which from easybuild.tools.environment import setvar from easybuild.tools.modules import get_software_root, get_software_version -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.utilities import nub @@ -63,8 +63,8 @@ def det_cmake_version(): regex = re.compile(r"^[cC][mM]ake version (?P[0-9]\.[0-9a-zA-Z.-]+)$", re.M) cmd = "cmake --version" - (out, _) = run_cmd(cmd, simple=False, log_ok=False, log_all=False, trace=False) - res = regex.search(out) + cmd_res = run_shell_cmd(cmd, hidden=True) + res = regex.search(cmd_res.output) if res: cmake_version = res.group('version') else: @@ -311,9 +311,9 @@ def configure_step(self, srcdir=None, builddir=None): self.cfg.get('configure_cmd'), self.cfg['configopts']]) - (out, _) = run_cmd(command, log_all=True, simple=False) + res = run_shell_cmd(command) - return out + return res.output def test_step(self): """CMake specific test setup""" From 93de2a94f6502bd43486e336dda2d49ba4da07b6 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 6 Dec 2023 13:23:51 +0000 Subject: [PATCH 074/145] Convert MesonNinja easyblock to run_shell_cmd. --- easybuild/easyblocks/generic/mesonninja.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/easybuild/easyblocks/generic/mesonninja.py b/easybuild/easyblocks/generic/mesonninja.py index 1fafaed53d..7297b84d4d 100644 --- a/easybuild/easyblocks/generic/mesonninja.py +++ b/easybuild/easyblocks/generic/mesonninja.py @@ -34,7 +34,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import change_dir, create_unused_dir, which from easybuild.tools.modules import get_software_version -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd DEFAULT_CONFIGURE_CMD = 'meson' DEFAULT_BUILD_CMD = 'ninja' @@ -99,8 +99,8 @@ def configure_step(self, cmd_prefix=''): 'preconfigopts': self.cfg['preconfigopts'], 'source_dir': build_dir, } - (out, _) = run_cmd(cmd, log_all=True, simple=False) - return out + res = run_shell_cmd(cmd) + return res.output def build_step(self, verbose=False, path=None): """ @@ -118,8 +118,8 @@ def build_step(self, verbose=False, path=None): 'parallel': parallel, 'prebuildopts': self.cfg['prebuildopts'], } - (out, _) = run_cmd(cmd, log_all=True, simple=False) - return out + res = run_shell_cmd(cmd) + return res.output def test_step(self): """ @@ -127,8 +127,8 @@ def test_step(self): """ if self.cfg['runtest']: cmd = "%s %s %s" % (self.cfg['pretestopts'], self.cfg['runtest'], self.cfg['testopts']) - (out, _) = run_cmd(cmd, log_all=True, simple=False) - return out + res = run_shell_cmd(cmd) + return res.output def install_step(self): """ @@ -146,5 +146,5 @@ def install_step(self): 'install_cmd': install_cmd, 'preinstallopts': self.cfg['preinstallopts'], } - (out, _) = run_cmd(cmd, log_all=True, simple=False) - return out + res = run_shell_cmd(cmd) + return res.output From a4485b46627bdeee6545e8f6ebb0065e9d199526 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 6 Dec 2023 13:26:35 +0000 Subject: [PATCH 075/145] Convert Binary easyblock to run_shell_cmd. --- easybuild/easyblocks/generic/binary.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/generic/binary.py b/easybuild/easyblocks/generic/binary.py index d9ff3a5bb7..62380d49cd 100644 --- a/easybuild/easyblocks/generic/binary.py +++ b/easybuild/easyblocks/generic/binary.py @@ -40,7 +40,7 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import adjust_permissions, copy_file, mkdir, remove_dir -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd PREPEND_TO_PATH_DEFAULT = [''] @@ -126,7 +126,7 @@ def install_step(self): for install_cmd in install_cmds: cmd = ' '.join([self.cfg['preinstallopts'], install_cmd, self.cfg['installopts']]) self.log.info("Running install command for %s: '%s'..." % (self.name, cmd)) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) else: raise EasyBuildError("Incorrect value type for install_cmds, should be list or tuple: ", install_cmds) From 4a3442f1c2a36da2ffcbbfcba41a56d745c4708a Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 6 Dec 2023 13:27:31 +0000 Subject: [PATCH 076/145] Convert CmdCp easyblock to run_shell_cmd. --- easybuild/easyblocks/generic/cmdcp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/generic/cmdcp.py b/easybuild/easyblocks/generic/cmdcp.py index 5a91112a4e..12c34486d2 100644 --- a/easybuild/easyblocks/generic/cmdcp.py +++ b/easybuild/easyblocks/generic/cmdcp.py @@ -32,7 +32,7 @@ from easybuild.easyblocks.generic.makecp import MakeCp from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class CmdCp(MakeCp): @@ -80,4 +80,4 @@ def build_step(self): raise EasyBuildError("No match for %s in %s, don't know which command to use.", src, self.cfg['cmds_map']) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) From d71b71e23310fd43506b0c41345c50c8d3686d3e Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Wed, 6 Dec 2023 14:08:31 +0000 Subject: [PATCH 077/145] Fix use of `out` in CMakeMake easyblock --- easybuild/easyblocks/generic/cmakemake.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/cmakemake.py b/easybuild/easyblocks/generic/cmakemake.py index 1552b082a6..030783df05 100644 --- a/easybuild/easyblocks/generic/cmakemake.py +++ b/easybuild/easyblocks/generic/cmakemake.py @@ -64,7 +64,8 @@ def det_cmake_version(): cmd = "cmake --version" cmd_res = run_shell_cmd(cmd, hidden=True) - res = regex.search(cmd_res.output) + out = cmd_res.output + res = regex.search(out) if res: cmake_version = res.group('version') else: From d9308581f191817346ff56c59b929f0ae1d40444 Mon Sep 17 00:00:00 2001 From: Chia-Jung Hsu Date: Wed, 11 Oct 2023 17:07:14 +0200 Subject: [PATCH 078/145] Replace all 'distutils.version' in easyblocks by 'easybuild.tools' --- easybuild/easyblocks/a/abaqus.py | 2 +- easybuild/easyblocks/a/acml.py | 2 +- easybuild/easyblocks/a/advisor.py | 2 +- easybuild/easyblocks/a/amber.py | 2 +- easybuild/easyblocks/a/ansys.py | 2 +- easybuild/easyblocks/a/aocc.py | 2 +- easybuild/easyblocks/a/armadillo.py | 2 +- easybuild/easyblocks/a/atlas.py | 2 +- easybuild/easyblocks/b/bamtools.py | 2 +- easybuild/easyblocks/b/bazel.py | 2 +- easybuild/easyblocks/b/berkeleygw.py | 2 +- easybuild/easyblocks/b/binutils.py | 2 +- easybuild/easyblocks/b/boost.py | 2 +- easybuild/easyblocks/b/bowtie.py | 2 +- easybuild/easyblocks/b/bowtie2.py | 2 +- easybuild/easyblocks/b/bwa.py | 2 +- easybuild/easyblocks/c/cgal.py | 2 +- easybuild/easyblocks/c/clang.py | 2 +- easybuild/easyblocks/c/clang_aomp.py | 2 +- easybuild/easyblocks/c/cp2k.py | 2 +- easybuild/easyblocks/c/cplex.py | 2 +- easybuild/easyblocks/c/cryptography.py | 2 +- easybuild/easyblocks/c/cuda.py | 2 +- easybuild/easyblocks/c/cudacompat.py | 2 +- easybuild/easyblocks/c/cudnn.py | 2 +- easybuild/easyblocks/d/dolfin.py | 2 +- easybuild/easyblocks/d/doxygen.py | 2 +- easybuild/easyblocks/e/easybuildmeta.py | 2 +- easybuild/easyblocks/e/eigen.py | 2 +- easybuild/easyblocks/e/elpa.py | 2 +- easybuild/easyblocks/e/esmf.py | 2 +- easybuild/easyblocks/f/ferret.py | 2 +- easybuild/easyblocks/f/fftw.py | 2 +- easybuild/easyblocks/f/flex.py | 2 +- easybuild/easyblocks/f/fluent.py | 2 +- easybuild/easyblocks/f/freesurfer.py | 2 +- easybuild/easyblocks/f/fsl.py | 2 +- easybuild/easyblocks/g/g2clib.py | 2 +- easybuild/easyblocks/g/gate.py | 2 +- easybuild/easyblocks/g/gcc.py | 2 +- easybuild/easyblocks/g/gctf.py | 2 +- easybuild/easyblocks/g/geant4.py | 2 +- easybuild/easyblocks/g/ghc.py | 2 +- easybuild/easyblocks/g/go.py | 2 +- easybuild/easyblocks/g/gromacs.py | 2 +- easybuild/easyblocks/generic/cmakemake.py | 2 +- easybuild/easyblocks/generic/gopackage.py | 2 +- easybuild/easyblocks/generic/intelbase.py | 2 +- easybuild/easyblocks/generic/juliapackage.py | 2 +- easybuild/easyblocks/generic/mesonninja.py | 2 +- easybuild/easyblocks/generic/pythonpackage.py | 2 +- easybuild/easyblocks/generic/rpm.py | 2 +- easybuild/easyblocks/generic/systemcompiler.py | 2 +- easybuild/easyblocks/h/healpix.py | 2 +- easybuild/easyblocks/h/hpcg.py | 2 +- easybuild/easyblocks/i/icc.py | 2 +- easybuild/easyblocks/i/ifort.py | 2 +- easybuild/easyblocks/i/imkl.py | 2 +- easybuild/easyblocks/i/impi.py | 2 +- easybuild/easyblocks/i/inspector.py | 2 +- easybuild/easyblocks/i/intel_compilers.py | 2 +- easybuild/easyblocks/i/ipp.py | 2 +- easybuild/easyblocks/i/itac.py | 2 +- easybuild/easyblocks/j/java.py | 2 +- easybuild/easyblocks/j/jaxlib.py | 2 +- easybuild/easyblocks/l/lammps.py | 2 +- easybuild/easyblocks/l/lapack.py | 2 +- easybuild/easyblocks/l/libint.py | 2 +- easybuild/easyblocks/l/libqglviewer.py | 2 +- easybuild/easyblocks/l/libsmm.py | 2 +- easybuild/easyblocks/l/libxml2.py | 2 +- easybuild/easyblocks/l/llvm.py | 2 +- easybuild/easyblocks/l/lua.py | 2 +- easybuild/easyblocks/m/mathematica.py | 2 +- easybuild/easyblocks/m/matlab.py | 2 +- easybuild/easyblocks/m/mcr.py | 2 +- easybuild/easyblocks/m/mesa.py | 2 +- easybuild/easyblocks/m/metis.py | 2 +- easybuild/easyblocks/m/molpro.py | 2 +- easybuild/easyblocks/m/mono.py | 2 +- easybuild/easyblocks/m/mothur.py | 2 +- easybuild/easyblocks/m/motioncor2.py | 2 +- easybuild/easyblocks/m/mpich.py | 2 +- easybuild/easyblocks/m/mrbayes.py | 2 +- easybuild/easyblocks/m/mrtrix.py | 2 +- easybuild/easyblocks/m/mumps.py | 2 +- easybuild/easyblocks/m/mvapich2.py | 2 +- easybuild/easyblocks/m/mxnet.py | 2 +- easybuild/easyblocks/m/mymedialite.py | 2 +- easybuild/easyblocks/n/namd.py | 2 +- easybuild/easyblocks/n/netcdf.py | 2 +- easybuild/easyblocks/n/neuron.py | 2 +- easybuild/easyblocks/n/numexpr.py | 2 +- easybuild/easyblocks/n/numpy.py | 2 +- easybuild/easyblocks/n/nvhpc.py | 2 +- easybuild/easyblocks/n/nwchem.py | 2 +- easybuild/easyblocks/o/ocaml.py | 2 +- easybuild/easyblocks/o/openbabel.py | 2 +- easybuild/easyblocks/o/openblas.py | 2 +- easybuild/easyblocks/o/opencv.py | 2 +- easybuild/easyblocks/o/openfoam.py | 2 +- easybuild/easyblocks/o/openmpi.py | 2 +- easybuild/easyblocks/o/openssl.py | 2 +- easybuild/easyblocks/o/openssl_wrapper.py | 2 +- easybuild/easyblocks/o/orca.py | 2 +- easybuild/easyblocks/p/paraver.py | 2 +- easybuild/easyblocks/p/parmetis.py | 2 +- easybuild/easyblocks/p/perl.py | 2 +- easybuild/easyblocks/p/petsc.py | 2 +- easybuild/easyblocks/p/pgi.py | 2 +- easybuild/easyblocks/p/picard.py | 2 +- easybuild/easyblocks/p/psi.py | 2 +- easybuild/easyblocks/p/psmpi.py | 2 +- easybuild/easyblocks/p/python.py | 2 +- easybuild/easyblocks/p/pytorch.py | 2 +- easybuild/easyblocks/q/qscintilla.py | 2 +- easybuild/easyblocks/q/qt.py | 2 +- easybuild/easyblocks/q/quantumespresso.py | 2 +- easybuild/easyblocks/r/r.py | 2 +- easybuild/easyblocks/r/repeatmasker.py | 2 +- easybuild/easyblocks/r/rmpi.py | 2 +- easybuild/easyblocks/r/root.py | 2 +- easybuild/easyblocks/s/samtools.py | 2 +- easybuild/easyblocks/s/scalapack.py | 2 +- easybuild/easyblocks/s/scalasca1.py | 2 +- easybuild/easyblocks/s/scipion.py | 2 +- easybuild/easyblocks/s/scipy.py | 2 +- easybuild/easyblocks/s/scotch.py | 2 +- easybuild/easyblocks/s/siesta.py | 2 +- easybuild/easyblocks/s/slepc.py | 2 +- easybuild/easyblocks/s/snphylo.py | 2 +- easybuild/easyblocks/s/suitesparse.py | 2 +- easybuild/easyblocks/s/superlu.py | 2 +- easybuild/easyblocks/t/tbb.py | 2 +- easybuild/easyblocks/t/tensorrt.py | 2 +- easybuild/easyblocks/t/tinker.py | 2 +- easybuild/easyblocks/t/tkinter.py | 2 +- easybuild/easyblocks/t/trilinos.py | 2 +- easybuild/easyblocks/t/trinity.py | 2 +- easybuild/easyblocks/u/ufc.py | 2 +- easybuild/easyblocks/v/vmd.py | 2 +- easybuild/easyblocks/v/vtune.py | 2 +- easybuild/easyblocks/w/wien2k.py | 2 +- easybuild/easyblocks/w/wps.py | 2 +- easybuild/easyblocks/w/wrf.py | 2 +- easybuild/easyblocks/w/wxpython.py | 2 +- easybuild/easyblocks/x/xmipp.py | 2 +- 147 files changed, 147 insertions(+), 147 deletions(-) diff --git a/easybuild/easyblocks/a/abaqus.py b/easybuild/easyblocks/a/abaqus.py index 50404d75db..371a1b2b42 100644 --- a/easybuild/easyblocks/a/abaqus.py +++ b/easybuild/easyblocks/a/abaqus.py @@ -32,9 +32,9 @@ @author: Jens Timmerman (Ghent University) @author: Simon Branford (University of Birmingham) """ -from distutils.version import LooseVersion import glob import os +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.binary import Binary from easybuild.framework.easyblock import EasyBlock diff --git a/easybuild/easyblocks/a/acml.py b/easybuild/easyblocks/a/acml.py index b20704a53e..5d82316dc5 100644 --- a/easybuild/easyblocks/a/acml.py +++ b/easybuild/easyblocks/a/acml.py @@ -33,7 +33,7 @@ """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.framework.easyblock import EasyBlock from easybuild.framework.easyconfig import CUSTOM diff --git a/easybuild/easyblocks/a/advisor.py b/easybuild/easyblocks/a/advisor.py index 54246495fa..45bced4c0a 100644 --- a/easybuild/easyblocks/a/advisor.py +++ b/easybuild/easyblocks/a/advisor.py @@ -31,7 +31,7 @@ """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.intelbase import IntelBase diff --git a/easybuild/easyblocks/a/amber.py b/easybuild/easyblocks/a/amber.py index 0140ac8d10..ca626883db 100644 --- a/easybuild/easyblocks/a/amber.py +++ b/easybuild/easyblocks/a/amber.py @@ -31,7 +31,7 @@ Enhanced/cleaned up by Kenneth Hoste (HPC-UGent) CMake support (Amber 20) added by James Carpenter and Simon Branford (University of Birmingham) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import os import easybuild.tools.environment as env diff --git a/easybuild/easyblocks/a/ansys.py b/easybuild/easyblocks/a/ansys.py index 0c4b834846..6e68fad40b 100644 --- a/easybuild/easyblocks/a/ansys.py +++ b/easybuild/easyblocks/a/ansys.py @@ -31,7 +31,7 @@ import os import re import stat -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.packedbinary import PackedBinary from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/a/aocc.py b/easybuild/easyblocks/a/aocc.py index 8ccdaf1d86..aff71ee1b0 100644 --- a/easybuild/easyblocks/a/aocc.py +++ b/easybuild/easyblocks/a/aocc.py @@ -34,7 +34,7 @@ import os import stat -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.packedbinary import PackedBinary from easybuild.framework.easyconfig import CUSTOM diff --git a/easybuild/easyblocks/a/armadillo.py b/easybuild/easyblocks/a/armadillo.py index a2faa04317..27415f3441 100644 --- a/easybuild/easyblocks/a/armadillo.py +++ b/easybuild/easyblocks/a/armadillo.py @@ -28,7 +28,7 @@ @author: Kenneth Hoste (Ghent University) """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.cmakemake import CMakeMake from easybuild.tools.build_log import EasyBuildError from easybuild.tools.modules import get_software_root diff --git a/easybuild/easyblocks/a/atlas.py b/easybuild/easyblocks/a/atlas.py index da8fdefbc9..2956e30636 100644 --- a/easybuild/easyblocks/a/atlas.py +++ b/easybuild/easyblocks/a/atlas.py @@ -36,7 +36,7 @@ import re import os import sys -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.framework.easyconfig import CUSTOM diff --git a/easybuild/easyblocks/b/bamtools.py b/easybuild/easyblocks/b/bamtools.py index 0a11f2e4cb..1938ce4c6a 100644 --- a/easybuild/easyblocks/b/bamtools.py +++ b/easybuild/easyblocks/b/bamtools.py @@ -28,7 +28,7 @@ @author: Andreas Panteli (The Cyprus Institute) @author: Kenneth Hoste (Ghent University) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.cmakemake import CMakeMake from easybuild.easyblocks.generic.makecp import MakeCp from easybuild.framework.easyconfig import CUSTOM diff --git a/easybuild/easyblocks/b/bazel.py b/easybuild/easyblocks/b/bazel.py index e626457001..97a013c8fb 100644 --- a/easybuild/easyblocks/b/bazel.py +++ b/easybuild/easyblocks/b/bazel.py @@ -25,7 +25,7 @@ """ EasyBuild support for building and installing Bazel, implemented as an easyblock """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import glob import os import tempfile diff --git a/easybuild/easyblocks/b/berkeleygw.py b/easybuild/easyblocks/b/berkeleygw.py index e9a9b87f8e..b5b2e1f28b 100644 --- a/easybuild/easyblocks/b/berkeleygw.py +++ b/easybuild/easyblocks/b/berkeleygw.py @@ -28,7 +28,7 @@ @author: Miguel Dias Costa (National University of Singapore) """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.generic.configuremake import ConfigureMake diff --git a/easybuild/easyblocks/b/binutils.py b/easybuild/easyblocks/b/binutils.py index 306a535d99..3985595288 100644 --- a/easybuild/easyblocks/b/binutils.py +++ b/easybuild/easyblocks/b/binutils.py @@ -30,7 +30,7 @@ import glob import os import re -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env from easybuild.easyblocks.generic.configuremake import ConfigureMake diff --git a/easybuild/easyblocks/b/boost.py b/easybuild/easyblocks/b/boost.py index 5110105cd8..7cb4bd7f03 100644 --- a/easybuild/easyblocks/b/boost.py +++ b/easybuild/easyblocks/b/boost.py @@ -38,7 +38,7 @@ @author: Michele Dolfi (ETH Zurich) @author: Simon Branford (University of Birmingham) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import fileinput import glob import os diff --git a/easybuild/easyblocks/b/bowtie.py b/easybuild/easyblocks/b/bowtie.py index 4daa8512ba..899019b35b 100644 --- a/easybuild/easyblocks/b/bowtie.py +++ b/easybuild/easyblocks/b/bowtie.py @@ -30,7 +30,7 @@ @author: Kenneth Hoste (Ghent University) @author: Jens Timmerman (Ghent University) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import glob import os import shutil diff --git a/easybuild/easyblocks/b/bowtie2.py b/easybuild/easyblocks/b/bowtie2.py index 285f9b98db..5d0821a6ed 100644 --- a/easybuild/easyblocks/b/bowtie2.py +++ b/easybuild/easyblocks/b/bowtie2.py @@ -16,7 +16,7 @@ @author: Fotis Georgatos (Uni.Lu) @author: Kenneth Hoste (Ghent University) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import os from easybuild.easyblocks.generic.makecp import MakeCp diff --git a/easybuild/easyblocks/b/bwa.py b/easybuild/easyblocks/b/bwa.py index 646cc85e4e..587dd03125 100644 --- a/easybuild/easyblocks/b/bwa.py +++ b/easybuild/easyblocks/b/bwa.py @@ -20,7 +20,7 @@ """ import os import glob -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/c/cgal.py b/easybuild/easyblocks/c/cgal.py index 6cb2937cd8..0d69d0543d 100644 --- a/easybuild/easyblocks/c/cgal.py +++ b/easybuild/easyblocks/c/cgal.py @@ -31,7 +31,7 @@ """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.cmakemake import CMakeMake from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/c/clang.py b/easybuild/easyblocks/c/clang.py index 3fb1b006f5..f66ddc5dde 100644 --- a/easybuild/easyblocks/c/clang.py +++ b/easybuild/easyblocks/c/clang.py @@ -38,7 +38,7 @@ import glob import os import shutil -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.cmakemake import CMakeMake from easybuild.framework.easyconfig import CUSTOM diff --git a/easybuild/easyblocks/c/clang_aomp.py b/easybuild/easyblocks/c/clang_aomp.py index 0104ab64f9..3a1a2081cf 100644 --- a/easybuild/easyblocks/c/clang_aomp.py +++ b/easybuild/easyblocks/c/clang_aomp.py @@ -31,7 +31,7 @@ import glob import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.clang import DEFAULT_TARGETS_MAP as LLVM_ARCH_MAP from easybuild.easyblocks.generic.bundle import Bundle from easybuild.framework.easyblock import EasyBlock diff --git a/easybuild/easyblocks/c/cp2k.py b/easybuild/easyblocks/c/cp2k.py index a921523e07..aa2ad4a93f 100644 --- a/easybuild/easyblocks/c/cp2k.py +++ b/easybuild/easyblocks/c/cp2k.py @@ -42,7 +42,7 @@ import re import os import sys -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.toolchain as toolchain from easybuild.framework.easyblock import EasyBlock diff --git a/easybuild/easyblocks/c/cplex.py b/easybuild/easyblocks/c/cplex.py index cc6ec0e52f..9c4a894a4e 100644 --- a/easybuild/easyblocks/c/cplex.py +++ b/easybuild/easyblocks/c/cplex.py @@ -31,7 +31,7 @@ @author: Pieter De Baets (Ghent University) @author: Jens Timmerman (Ghent University) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import glob import os import stat diff --git a/easybuild/easyblocks/c/cryptography.py b/easybuild/easyblocks/c/cryptography.py index bfb6688358..ac0d4c68d1 100644 --- a/easybuild/easyblocks/c/cryptography.py +++ b/easybuild/easyblocks/c/cryptography.py @@ -27,7 +27,7 @@ @author: Alexander Grund """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.pythonpackage import PythonPackage from easybuild.tools.run import run_cmd diff --git a/easybuild/easyblocks/c/cuda.py b/easybuild/easyblocks/c/cuda.py index 32629aab40..68aa61bf21 100644 --- a/easybuild/easyblocks/c/cuda.py +++ b/easybuild/easyblocks/c/cuda.py @@ -38,7 +38,7 @@ import re import stat -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.binary import Binary from easybuild.framework.easyconfig import CUSTOM diff --git a/easybuild/easyblocks/c/cudacompat.py b/easybuild/easyblocks/c/cudacompat.py index e97db8e2c0..ef72014175 100644 --- a/easybuild/easyblocks/c/cudacompat.py +++ b/easybuild/easyblocks/c/cudacompat.py @@ -31,7 +31,7 @@ """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.binary import Binary from easybuild.framework.easyconfig import CUSTOM, MANDATORY diff --git a/easybuild/easyblocks/c/cudnn.py b/easybuild/easyblocks/c/cudnn.py index 39e04fe8d5..d22834985d 100644 --- a/easybuild/easyblocks/c/cudnn.py +++ b/easybuild/easyblocks/c/cudnn.py @@ -28,7 +28,7 @@ @author: Simon Branford (University of Birmingham) @author: Robert Mijakovic (LuxProvide) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.tarball import Tarball from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/d/dolfin.py b/easybuild/easyblocks/d/dolfin.py index 6cb323636c..086fdff170 100644 --- a/easybuild/easyblocks/d/dolfin.py +++ b/easybuild/easyblocks/d/dolfin.py @@ -32,7 +32,7 @@ import os import re import tempfile -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain diff --git a/easybuild/easyblocks/d/doxygen.py b/easybuild/easyblocks/d/doxygen.py index 7a8a16365d..498e4f920b 100644 --- a/easybuild/easyblocks/d/doxygen.py +++ b/easybuild/easyblocks/d/doxygen.py @@ -33,7 +33,7 @@ @author: Balazs Hajgato (Free University Brussels (VUB)) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.tools.run import run_cmd from easybuild.easyblocks.generic.cmakemake import CMakeMake diff --git a/easybuild/easyblocks/e/easybuildmeta.py b/easybuild/easyblocks/e/easybuildmeta.py index f4b52ff921..73e1115fae 100644 --- a/easybuild/easyblocks/e/easybuildmeta.py +++ b/easybuild/easyblocks/e/easybuildmeta.py @@ -31,7 +31,7 @@ import os import re import sys -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.pythonpackage import PythonPackage, det_pip_version from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/e/eigen.py b/easybuild/easyblocks/e/eigen.py index 70dc6a4e36..70205c5ee0 100644 --- a/easybuild/easyblocks/e/eigen.py +++ b/easybuild/easyblocks/e/eigen.py @@ -19,7 +19,7 @@ import os import shutil -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.cmakemake import CMakeMake from easybuild.tools.filetools import copy_dir, copy_file, mkdir, apply_regex_substitutions diff --git a/easybuild/easyblocks/e/elpa.py b/easybuild/easyblocks/e/elpa.py index 41b81e15e8..44acaf507b 100644 --- a/easybuild/easyblocks/e/elpa.py +++ b/easybuild/easyblocks/e/elpa.py @@ -30,7 +30,7 @@ @author: Kenneth Hoste (Ghent University) """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env from easybuild.easyblocks.generic.configuremake import ConfigureMake diff --git a/easybuild/easyblocks/e/esmf.py b/easybuild/easyblocks/e/esmf.py index 03ff400619..676be44e44 100644 --- a/easybuild/easyblocks/e/esmf.py +++ b/easybuild/easyblocks/e/esmf.py @@ -30,7 +30,7 @@ @author: Maxime Boissonneault (Digital Research Alliance of Canada) """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain diff --git a/easybuild/easyblocks/f/ferret.py b/easybuild/easyblocks/f/ferret.py index 37e38e9a0b..8bde8f5641 100644 --- a/easybuild/easyblocks/f/ferret.py +++ b/easybuild/easyblocks/f/ferret.py @@ -36,7 +36,7 @@ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/f/fftw.py b/easybuild/easyblocks/f/fftw.py index 6f10c711c9..aa30f3bf06 100644 --- a/easybuild/easyblocks/f/fftw.py +++ b/easybuild/easyblocks/f/fftw.py @@ -27,7 +27,7 @@ @author: Kenneth Hoste (HPC-UGent) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.generic.configuremake import ConfigureMake diff --git a/easybuild/easyblocks/f/flex.py b/easybuild/easyblocks/f/flex.py index a7fd414f8d..8e17da560b 100644 --- a/easybuild/easyblocks/f/flex.py +++ b/easybuild/easyblocks/f/flex.py @@ -27,7 +27,7 @@ @author: Kenneth Hoste (Ghent University) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import os from easybuild.easyblocks.generic.configuremake import ConfigureMake diff --git a/easybuild/easyblocks/f/fluent.py b/easybuild/easyblocks/f/fluent.py index 5976f8ea4f..5b2a479ce4 100644 --- a/easybuild/easyblocks/f/fluent.py +++ b/easybuild/easyblocks/f/fluent.py @@ -29,7 +29,7 @@ """ import os import stat -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.packedbinary import PackedBinary from easybuild.framework.easyconfig import CUSTOM diff --git a/easybuild/easyblocks/f/freesurfer.py b/easybuild/easyblocks/f/freesurfer.py index cccb0fbd2b..e547c49453 100644 --- a/easybuild/easyblocks/f/freesurfer.py +++ b/easybuild/easyblocks/f/freesurfer.py @@ -30,7 +30,7 @@ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.tarball import Tarball from easybuild.framework.easyconfig import MANDATORY diff --git a/easybuild/easyblocks/f/fsl.py b/easybuild/easyblocks/f/fsl.py index e8ebaa9b21..dbd95edce5 100644 --- a/easybuild/easyblocks/f/fsl.py +++ b/easybuild/easyblocks/f/fsl.py @@ -31,7 +31,7 @@ import difflib import os import re -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env from easybuild.framework.easyblock import EasyBlock diff --git a/easybuild/easyblocks/g/g2clib.py b/easybuild/easyblocks/g/g2clib.py index 15ad20f57b..591280cd0f 100644 --- a/easybuild/easyblocks/g/g2clib.py +++ b/easybuild/easyblocks/g/g2clib.py @@ -40,7 +40,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.modules import get_software_root -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion class EB_g2clib(ConfigureMake): diff --git a/easybuild/easyblocks/g/gate.py b/easybuild/easyblocks/g/gate.py index 572204b836..223f49eefe 100644 --- a/easybuild/easyblocks/g/gate.py +++ b/easybuild/easyblocks/g/gate.py @@ -34,7 +34,7 @@ """ import os import shutil -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.generic.cmakemake import CMakeMake diff --git a/easybuild/easyblocks/g/gcc.py b/easybuild/easyblocks/g/gcc.py index 7ee469763e..589bdb5386 100644 --- a/easybuild/easyblocks/g/gcc.py +++ b/easybuild/easyblocks/g/gcc.py @@ -39,7 +39,7 @@ import re import shutil from copy import copy -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env from easybuild.easyblocks.clang import DEFAULT_TARGETS_MAP as LLVM_ARCH_MAP diff --git a/easybuild/easyblocks/g/gctf.py b/easybuild/easyblocks/g/gctf.py index 9e2ca40b86..17397a08c9 100644 --- a/easybuild/easyblocks/g/gctf.py +++ b/easybuild/easyblocks/g/gctf.py @@ -36,7 +36,7 @@ from easybuild.tools.filetools import adjust_permissions, copy_file, mkdir from easybuild.tools.filetools import symlink, write_file from easybuild.tools.modules import get_software_root -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion class EB_Gctf(EasyBlock): diff --git a/easybuild/easyblocks/g/geant4.py b/easybuild/easyblocks/g/geant4.py index ca431ca946..6c94076980 100644 --- a/easybuild/easyblocks/g/geant4.py +++ b/easybuild/easyblocks/g/geant4.py @@ -35,7 +35,7 @@ import os import shutil import re -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env from easybuild.framework.easyconfig import CUSTOM diff --git a/easybuild/easyblocks/g/ghc.py b/easybuild/easyblocks/g/ghc.py index 20c6edcd75..af2ef6382f 100644 --- a/easybuild/easyblocks/g/ghc.py +++ b/easybuild/easyblocks/g/ghc.py @@ -27,7 +27,7 @@ @author: Andy Georges (Ghent University) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake diff --git a/easybuild/easyblocks/g/go.py b/easybuild/easyblocks/g/go.py index 932df72f3e..97f0d79db7 100644 --- a/easybuild/easyblocks/g/go.py +++ b/easybuild/easyblocks/g/go.py @@ -31,7 +31,7 @@ import os import shutil -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/g/gromacs.py b/easybuild/easyblocks/g/gromacs.py index 973b3f323a..74d129626c 100644 --- a/easybuild/easyblocks/g/gromacs.py +++ b/easybuild/easyblocks/g/gromacs.py @@ -37,7 +37,7 @@ import os import re import shutil -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain diff --git a/easybuild/easyblocks/generic/cmakemake.py b/easybuild/easyblocks/generic/cmakemake.py index 0401f872e4..0a326a8e5a 100644 --- a/easybuild/easyblocks/generic/cmakemake.py +++ b/easybuild/easyblocks/generic/cmakemake.py @@ -36,7 +36,7 @@ import glob import re import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.framework.easyconfig import BUILD, CUSTOM diff --git a/easybuild/easyblocks/generic/gopackage.py b/easybuild/easyblocks/generic/gopackage.py index 5c2d96bdc9..8a3b591402 100644 --- a/easybuild/easyblocks/generic/gopackage.py +++ b/easybuild/easyblocks/generic/gopackage.py @@ -28,7 +28,7 @@ @author: Pavel Grochal (INUITS) """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env from easybuild.framework.easyblock import EasyBlock diff --git a/easybuild/easyblocks/generic/intelbase.py b/easybuild/easyblocks/generic/intelbase.py index 142f69cfd8..9a9fa9e32b 100644 --- a/easybuild/easyblocks/generic/intelbase.py +++ b/easybuild/easyblocks/generic/intelbase.py @@ -40,7 +40,7 @@ import shutil import stat import tempfile -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env from easybuild.framework.easyblock import EasyBlock diff --git a/easybuild/easyblocks/generic/juliapackage.py b/easybuild/easyblocks/generic/juliapackage.py index 6e535d7f44..2ff1e49e89 100644 --- a/easybuild/easyblocks/generic/juliapackage.py +++ b/easybuild/easyblocks/generic/juliapackage.py @@ -30,7 +30,7 @@ import os import re -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env from easybuild.framework.easyconfig import CUSTOM diff --git a/easybuild/easyblocks/generic/mesonninja.py b/easybuild/easyblocks/generic/mesonninja.py index 8ce6b530e4..1fafaed53d 100644 --- a/easybuild/easyblocks/generic/mesonninja.py +++ b/easybuild/easyblocks/generic/mesonninja.py @@ -28,7 +28,7 @@ @author: Kenneth Hoste (Ghent University) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.framework.easyblock import EasyBlock from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 29c7a8d803..63bb54b527 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -37,7 +37,7 @@ import re import sys import tempfile -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from distutils.sysconfig import get_config_vars import easybuild.tools.environment as env diff --git a/easybuild/easyblocks/generic/rpm.py b/easybuild/easyblocks/generic/rpm.py index 0a3f284c7f..1ab63bfff1 100644 --- a/easybuild/easyblocks/generic/rpm.py +++ b/easybuild/easyblocks/generic/rpm.py @@ -37,7 +37,7 @@ import os import re import tempfile -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from os.path import expanduser import easybuild.tools.environment as env diff --git a/easybuild/easyblocks/generic/systemcompiler.py b/easybuild/easyblocks/generic/systemcompiler.py index def79c8543..0b01c30036 100644 --- a/easybuild/easyblocks/generic/systemcompiler.py +++ b/easybuild/easyblocks/generic/systemcompiler.py @@ -31,7 +31,7 @@ """ import os import re -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.base import fancylogger from easybuild.easyblocks.generic.bundle import Bundle diff --git a/easybuild/easyblocks/h/healpix.py b/easybuild/easyblocks/h/healpix.py index badc564abb..3f3bd08b2d 100644 --- a/easybuild/easyblocks/h/healpix.py +++ b/easybuild/easyblocks/h/healpix.py @@ -29,7 +29,7 @@ @author: Josef Dvoracek (Institute of Physics, Czech Academy of Sciences) """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.generic.configuremake import ConfigureMake diff --git a/easybuild/easyblocks/h/hpcg.py b/easybuild/easyblocks/h/hpcg.py index 8d68fa34ea..13d9fe0621 100644 --- a/easybuild/easyblocks/h/hpcg.py +++ b/easybuild/easyblocks/h/hpcg.py @@ -37,7 +37,7 @@ from easybuild.tools.config import build_option from easybuild.tools.filetools import mkdir from easybuild.tools.run import run_cmd -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion class EB_HPCG(ConfigureMake): diff --git a/easybuild/easyblocks/i/icc.py b/easybuild/easyblocks/i/icc.py index 37651f9e30..f827dd4679 100644 --- a/easybuild/easyblocks/i/icc.py +++ b/easybuild/easyblocks/i/icc.py @@ -36,7 +36,7 @@ import os import re -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.intelbase import IntelBase, ACTIVATION_NAME_2012, COMP_ALL from easybuild.easyblocks.generic.intelbase import LICENSE_FILE_NAME_2012 diff --git a/easybuild/easyblocks/i/ifort.py b/easybuild/easyblocks/i/ifort.py index d47013a510..5b3356c84d 100644 --- a/easybuild/easyblocks/i/ifort.py +++ b/easybuild/easyblocks/i/ifort.py @@ -34,7 +34,7 @@ """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.intelbase import IntelBase from easybuild.easyblocks.icc import EB_icc # @UnresolvedImport diff --git a/easybuild/easyblocks/i/imkl.py b/easybuild/easyblocks/i/imkl.py index a342a4686c..2e855b4418 100644 --- a/easybuild/easyblocks/i/imkl.py +++ b/easybuild/easyblocks/i/imkl.py @@ -39,7 +39,7 @@ import os import shutil import tempfile -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain diff --git a/easybuild/easyblocks/i/impi.py b/easybuild/easyblocks/i/impi.py index 7eee0df099..98cac7e5bf 100644 --- a/easybuild/easyblocks/i/impi.py +++ b/easybuild/easyblocks/i/impi.py @@ -34,7 +34,7 @@ @author: Alex Domingo (Vrije Universiteit Brussel) """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.generic.intelbase import IntelBase, ACTIVATION_NAME_2012, LICENSE_FILE_NAME_2012 diff --git a/easybuild/easyblocks/i/inspector.py b/easybuild/easyblocks/i/inspector.py index e3ce7653b1..a0d595c2e8 100644 --- a/easybuild/easyblocks/i/inspector.py +++ b/easybuild/easyblocks/i/inspector.py @@ -29,7 +29,7 @@ @author: Damian Alvarez (Forschungzentrum Juelich GmbH) """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.intelbase import IntelBase, ACTIVATION_NAME_2012, LICENSE_FILE_NAME_2012 diff --git a/easybuild/easyblocks/i/intel_compilers.py b/easybuild/easyblocks/i/intel_compilers.py index 4ed76e2f25..d1839a7a7b 100644 --- a/easybuild/easyblocks/i/intel_compilers.py +++ b/easybuild/easyblocks/i/intel_compilers.py @@ -28,7 +28,7 @@ @author: Kenneth Hoste (Ghent University) """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.intelbase import IntelBase from easybuild.easyblocks.t.tbb import get_tbb_gccprefix diff --git a/easybuild/easyblocks/i/ipp.py b/easybuild/easyblocks/i/ipp.py index 789a44d71c..3b70620b0d 100644 --- a/easybuild/easyblocks/i/ipp.py +++ b/easybuild/easyblocks/i/ipp.py @@ -34,7 +34,7 @@ @author: Damian Alvarez (Forschungszentrum Juelich GmbH) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import os from easybuild.easyblocks.generic.intelbase import IntelBase, ACTIVATION_NAME_2012, LICENSE_FILE_NAME_2012 diff --git a/easybuild/easyblocks/i/itac.py b/easybuild/easyblocks/i/itac.py index 1f37b8ab01..542b1f5e37 100644 --- a/easybuild/easyblocks/i/itac.py +++ b/easybuild/easyblocks/i/itac.py @@ -34,7 +34,7 @@ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.intelbase import IntelBase from easybuild.framework.easyconfig import CUSTOM diff --git a/easybuild/easyblocks/j/java.py b/easybuild/easyblocks/j/java.py index 9eca14cf72..caef94fabe 100644 --- a/easybuild/easyblocks/j/java.py +++ b/easybuild/easyblocks/j/java.py @@ -32,7 +32,7 @@ import os import stat -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.packedbinary import PackedBinary from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option diff --git a/easybuild/easyblocks/j/jaxlib.py b/easybuild/easyblocks/j/jaxlib.py index dac39bf16c..ad6fa016f5 100644 --- a/easybuild/easyblocks/j/jaxlib.py +++ b/easybuild/easyblocks/j/jaxlib.py @@ -33,7 +33,7 @@ import os import tempfile -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env from easybuild.easyblocks.generic.pythonpackage import PythonPackage diff --git a/easybuild/easyblocks/l/lammps.py b/easybuild/easyblocks/l/lammps.py index 8b4b4dbe2e..2769014eab 100644 --- a/easybuild/easyblocks/l/lammps.py +++ b/easybuild/easyblocks/l/lammps.py @@ -34,7 +34,7 @@ import os import re import tempfile -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain diff --git a/easybuild/easyblocks/l/lapack.py b/easybuild/easyblocks/l/lapack.py index 112875a8fa..b0322030b7 100644 --- a/easybuild/easyblocks/l/lapack.py +++ b/easybuild/easyblocks/l/lapack.py @@ -31,7 +31,7 @@ @author: Pieter De Baets (Ghent University) @author: Jens Timmerman (Ghent University) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import glob import os diff --git a/easybuild/easyblocks/l/libint.py b/easybuild/easyblocks/l/libint.py index aa76bb6020..a5ff58df41 100644 --- a/easybuild/easyblocks/l/libint.py +++ b/easybuild/easyblocks/l/libint.py @@ -30,7 +30,7 @@ """ import os.path -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import DEFAULT_CONFIGURE_CMD, ConfigureMake from easybuild.easyblocks.generic.cmakemake import CMakeMake diff --git a/easybuild/easyblocks/l/libqglviewer.py b/easybuild/easyblocks/l/libqglviewer.py index 55d6a5d0a6..b126068ead 100644 --- a/easybuild/easyblocks/l/libqglviewer.py +++ b/easybuild/easyblocks/l/libqglviewer.py @@ -33,7 +33,7 @@ from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.modules import get_software_root from easybuild.tools.build_log import EasyBuildError -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion class EB_libQGLViewer(ConfigureMake): diff --git a/easybuild/easyblocks/l/libsmm.py b/easybuild/easyblocks/l/libsmm.py index 19a9f6c145..58d73cbb45 100644 --- a/easybuild/easyblocks/l/libsmm.py +++ b/easybuild/easyblocks/l/libsmm.py @@ -33,7 +33,7 @@ """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.toolchain as toolchain from easybuild.framework.easyblock import EasyBlock diff --git a/easybuild/easyblocks/l/libxml2.py b/easybuild/easyblocks/l/libxml2.py index 73e83cc804..10544c2db9 100644 --- a/easybuild/easyblocks/l/libxml2.py +++ b/easybuild/easyblocks/l/libxml2.py @@ -30,7 +30,7 @@ @author: Alan O'Cais (Juelich Supercomputing Centre) @author: Kenneth Hoste (Ghent University) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import os import easybuild.tools.environment as env diff --git a/easybuild/easyblocks/l/llvm.py b/easybuild/easyblocks/l/llvm.py index 5bced9a4c2..7b63296045 100644 --- a/easybuild/easyblocks/l/llvm.py +++ b/easybuild/easyblocks/l/llvm.py @@ -37,7 +37,7 @@ from easybuild.tools.filetools import move_file from easybuild.tools.modules import get_software_root from easybuild.tools.systemtools import get_cpu_architecture -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion class EB_LLVM(CMakeMake): diff --git a/easybuild/easyblocks/l/lua.py b/easybuild/easyblocks/l/lua.py index 16fc4065e1..e77c44775c 100644 --- a/easybuild/easyblocks/l/lua.py +++ b/easybuild/easyblocks/l/lua.py @@ -28,7 +28,7 @@ @author: Ruben Di Battista (Ecole Polytechnique) @author: Kenneth Hoste (Ghent University) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import os from easybuild.easyblocks.generic.configuremake import ConfigureMake diff --git a/easybuild/easyblocks/m/mathematica.py b/easybuild/easyblocks/m/mathematica.py index 9f1fb78827..6f18474490 100644 --- a/easybuild/easyblocks/m/mathematica.py +++ b/easybuild/easyblocks/m/mathematica.py @@ -28,7 +28,7 @@ @author: Kenneth Hoste (Ghent University) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import glob import os diff --git a/easybuild/easyblocks/m/matlab.py b/easybuild/easyblocks/m/matlab.py index ec2f43fa39..7e5a7e54be 100644 --- a/easybuild/easyblocks/m/matlab.py +++ b/easybuild/easyblocks/m/matlab.py @@ -37,7 +37,7 @@ import stat import tempfile -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.packedbinary import PackedBinary from easybuild.framework.easyconfig import CUSTOM diff --git a/easybuild/easyblocks/m/mcr.py b/easybuild/easyblocks/m/mcr.py index 49cb3b36f4..7a420bd660 100644 --- a/easybuild/easyblocks/m/mcr.py +++ b/easybuild/easyblocks/m/mcr.py @@ -38,7 +38,7 @@ import re import shutil import stat -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.packedbinary import PackedBinary from easybuild.framework.easyconfig import CUSTOM diff --git a/easybuild/easyblocks/m/mesa.py b/easybuild/easyblocks/m/mesa.py index 897faacbaa..b6d127361f 100644 --- a/easybuild/easyblocks/m/mesa.py +++ b/easybuild/easyblocks/m/mesa.py @@ -31,7 +31,7 @@ @author: Alexander Grund (TU Dresden) """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.mesonninja import MesonNinja from easybuild.tools.filetools import copy_dir diff --git a/easybuild/easyblocks/m/metis.py b/easybuild/easyblocks/m/metis.py index b8582b7842..451ca67f84 100644 --- a/easybuild/easyblocks/m/metis.py +++ b/easybuild/easyblocks/m/metis.py @@ -33,7 +33,7 @@ """ import os import shutil -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/m/molpro.py b/easybuild/easyblocks/m/molpro.py index 86358f0dcd..6cfdfa069d 100644 --- a/easybuild/easyblocks/m/molpro.py +++ b/easybuild/easyblocks/m/molpro.py @@ -30,7 +30,7 @@ import os import shutil import re -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.binary import Binary from easybuild.easyblocks.generic.configuremake import ConfigureMake diff --git a/easybuild/easyblocks/m/mono.py b/easybuild/easyblocks/m/mono.py index b23088ba65..11b04cce53 100644 --- a/easybuild/easyblocks/m/mono.py +++ b/easybuild/easyblocks/m/mono.py @@ -31,7 +31,7 @@ @author: Pieter De Baets (Ghent University) @author: Jens Timmerman (Ghent University) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import os import shutil diff --git a/easybuild/easyblocks/m/mothur.py b/easybuild/easyblocks/m/mothur.py index 78f5f0f832..bc14dd4cef 100644 --- a/easybuild/easyblocks/m/mothur.py +++ b/easybuild/easyblocks/m/mothur.py @@ -30,7 +30,7 @@ import glob import os import shutil -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/m/motioncor2.py b/easybuild/easyblocks/m/motioncor2.py index 9224e89d9b..7b0c086a04 100644 --- a/easybuild/easyblocks/m/motioncor2.py +++ b/easybuild/easyblocks/m/motioncor2.py @@ -32,7 +32,7 @@ import os import stat -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.packedbinary import PackedBinary from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import adjust_permissions, copy_file, mkdir, write_file diff --git a/easybuild/easyblocks/m/mpich.py b/easybuild/easyblocks/m/mpich.py index df97d7265b..05057f4ff2 100644 --- a/easybuild/easyblocks/m/mpich.py +++ b/easybuild/easyblocks/m/mpich.py @@ -34,7 +34,7 @@ @author: Xavier Besseron (University of Luxembourg) """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env from easybuild.easyblocks.generic.configuremake import ConfigureMake diff --git a/easybuild/easyblocks/m/mrbayes.py b/easybuild/easyblocks/m/mrbayes.py index 8fc399b5d3..ea7da301b5 100644 --- a/easybuild/easyblocks/m/mrbayes.py +++ b/easybuild/easyblocks/m/mrbayes.py @@ -36,7 +36,7 @@ """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/m/mrtrix.py b/easybuild/easyblocks/m/mrtrix.py index 7b148131a8..a676719c65 100644 --- a/easybuild/easyblocks/m/mrtrix.py +++ b/easybuild/easyblocks/m/mrtrix.py @@ -27,7 +27,7 @@ """ import glob import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env from easybuild.framework.easyblock import EasyBlock diff --git a/easybuild/easyblocks/m/mumps.py b/easybuild/easyblocks/m/mumps.py index 26c43adbf9..d701f82de6 100644 --- a/easybuild/easyblocks/m/mumps.py +++ b/easybuild/easyblocks/m/mumps.py @@ -34,7 +34,7 @@ import os import shutil -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.tools import toolchain diff --git a/easybuild/easyblocks/m/mvapich2.py b/easybuild/easyblocks/m/mvapich2.py index 3aecd40103..5810f227ac 100644 --- a/easybuild/easyblocks/m/mvapich2.py +++ b/easybuild/easyblocks/m/mvapich2.py @@ -34,7 +34,7 @@ @author: Xavier Besseron (University of Luxembourg) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.mpich import EB_MPICH from easybuild.framework.easyconfig import CUSTOM diff --git a/easybuild/easyblocks/m/mxnet.py b/easybuild/easyblocks/m/mxnet.py index 10bf26cc5d..8e52f3635d 100644 --- a/easybuild/easyblocks/m/mxnet.py +++ b/easybuild/easyblocks/m/mxnet.py @@ -30,7 +30,7 @@ import glob import os import shutil -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env from easybuild.easyblocks.generic.makecp import MakeCp diff --git a/easybuild/easyblocks/m/mymedialite.py b/easybuild/easyblocks/m/mymedialite.py index d4876e2d31..181ef9f051 100644 --- a/easybuild/easyblocks/m/mymedialite.py +++ b/easybuild/easyblocks/m/mymedialite.py @@ -32,7 +32,7 @@ @author: Jens Timmerman (Ghent University) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.tools.run import run_cmd diff --git a/easybuild/easyblocks/n/namd.py b/easybuild/easyblocks/n/namd.py index a0f4c82b95..301dfb5028 100644 --- a/easybuild/easyblocks/n/namd.py +++ b/easybuild/easyblocks/n/namd.py @@ -17,7 +17,7 @@ import os import re import shutil -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.generic.makecp import MakeCp diff --git a/easybuild/easyblocks/n/netcdf.py b/easybuild/easyblocks/n/netcdf.py index 45017748bb..b1b0a41ed8 100644 --- a/easybuild/easyblocks/n/netcdf.py +++ b/easybuild/easyblocks/n/netcdf.py @@ -33,7 +33,7 @@ """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain diff --git a/easybuild/easyblocks/n/neuron.py b/easybuild/easyblocks/n/neuron.py index a094670be6..cdad4158e5 100644 --- a/easybuild/easyblocks/n/neuron.py +++ b/easybuild/easyblocks/n/neuron.py @@ -41,7 +41,7 @@ from easybuild.tools.run import run_cmd from easybuild.tools.systemtools import get_shared_lib_ext -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion class EB_NEURON(CMakeMake): diff --git a/easybuild/easyblocks/n/numexpr.py b/easybuild/easyblocks/n/numexpr.py index 1c62b4502c..e5d192fff2 100644 --- a/easybuild/easyblocks/n/numexpr.py +++ b/easybuild/easyblocks/n/numexpr.py @@ -26,7 +26,7 @@ EasyBuild support for building and installing numexpr, implemented as an easyblock """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.pythonpackage import PythonPackage from easybuild.tools.filetools import write_file diff --git a/easybuild/easyblocks/n/numpy.py b/easybuild/easyblocks/n/numpy.py index 734953382c..8f5a049e0d 100644 --- a/easybuild/easyblocks/n/numpy.py +++ b/easybuild/easyblocks/n/numpy.py @@ -45,7 +45,7 @@ from easybuild.tools.filetools import change_dir, mkdir, read_file, remove_dir from easybuild.tools.modules import get_software_root from easybuild.tools.run import run_cmd -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion class EB_numpy(FortranPythonPackage): diff --git a/easybuild/easyblocks/n/nvhpc.py b/easybuild/easyblocks/n/nvhpc.py index b9b27cb009..6efc018a93 100644 --- a/easybuild/easyblocks/n/nvhpc.py +++ b/easybuild/easyblocks/n/nvhpc.py @@ -40,7 +40,7 @@ import sys import platform -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.packedbinary import PackedBinary from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.filetools import adjust_permissions, write_file diff --git a/easybuild/easyblocks/n/nwchem.py b/easybuild/easyblocks/n/nwchem.py index 667b0bbd1e..2e212d0a66 100644 --- a/easybuild/easyblocks/n/nwchem.py +++ b/easybuild/easyblocks/n/nwchem.py @@ -36,7 +36,7 @@ import easybuild.tools.config as config import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/o/ocaml.py b/easybuild/easyblocks/o/ocaml.py index fb43648798..0a8d9c8172 100644 --- a/easybuild/easyblocks/o/ocaml.py +++ b/easybuild/easyblocks/o/ocaml.py @@ -29,7 +29,7 @@ """ import os import re -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/o/openbabel.py b/easybuild/easyblocks/o/openbabel.py index b054cc56b0..61e54e2cc5 100644 --- a/easybuild/easyblocks/o/openbabel.py +++ b/easybuild/easyblocks/o/openbabel.py @@ -36,7 +36,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.modules import get_software_root, get_software_version from easybuild.tools.systemtools import get_shared_lib_ext -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion class EB_OpenBabel(CMakeMake): diff --git a/easybuild/easyblocks/o/openblas.py b/easybuild/easyblocks/o/openblas.py index 996dd280e1..e4b5e61bac 100644 --- a/easybuild/easyblocks/o/openblas.py +++ b/easybuild/easyblocks/o/openblas.py @@ -7,7 +7,7 @@ """ import os import re -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.systemtools import POWER, get_cpu_architecture, get_shared_lib_ext diff --git a/easybuild/easyblocks/o/opencv.py b/easybuild/easyblocks/o/opencv.py index 69a97a903d..538d7a7e29 100644 --- a/easybuild/easyblocks/o/opencv.py +++ b/easybuild/easyblocks/o/opencv.py @@ -30,7 +30,7 @@ """ import glob import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.cmakemake import CMakeMake from easybuild.easyblocks.generic.pythonpackage import det_pylibdir diff --git a/easybuild/easyblocks/o/openfoam.py b/easybuild/easyblocks/o/openfoam.py index 68589af669..a13368c0ef 100644 --- a/easybuild/easyblocks/o/openfoam.py +++ b/easybuild/easyblocks/o/openfoam.py @@ -41,7 +41,7 @@ import shutil import stat import tempfile -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain diff --git a/easybuild/easyblocks/o/openmpi.py b/easybuild/easyblocks/o/openmpi.py index e76dd131bf..b60c01142c 100644 --- a/easybuild/easyblocks/o/openmpi.py +++ b/easybuild/easyblocks/o/openmpi.py @@ -30,7 +30,7 @@ """ import os import re -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.generic.configuremake import ConfigureMake diff --git a/easybuild/easyblocks/o/openssl.py b/easybuild/easyblocks/o/openssl.py index 8238772899..c84a6e065e 100644 --- a/easybuild/easyblocks/o/openssl.py +++ b/easybuild/easyblocks/o/openssl.py @@ -33,7 +33,7 @@ import os import re -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.framework.easyconfig import CUSTOM diff --git a/easybuild/easyblocks/o/openssl_wrapper.py b/easybuild/easyblocks/o/openssl_wrapper.py index f73a5e5295..aa4a5dd91c 100644 --- a/easybuild/easyblocks/o/openssl_wrapper.py +++ b/easybuild/easyblocks/o/openssl_wrapper.py @@ -30,7 +30,7 @@ import os import re -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.bundle import Bundle from easybuild.framework.easyconfig import CUSTOM diff --git a/easybuild/easyblocks/o/orca.py b/easybuild/easyblocks/o/orca.py index e6066a28eb..3f08570d81 100644 --- a/easybuild/easyblocks/o/orca.py +++ b/easybuild/easyblocks/o/orca.py @@ -30,7 +30,7 @@ import glob import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.makecp import MakeCp from easybuild.easyblocks.generic.packedbinary import PackedBinary from easybuild.framework.easyconfig import CUSTOM diff --git a/easybuild/easyblocks/p/paraver.py b/easybuild/easyblocks/p/paraver.py index 181d660707..08288d2721 100644 --- a/easybuild/easyblocks/p/paraver.py +++ b/easybuild/easyblocks/p/paraver.py @@ -31,7 +31,7 @@ """ import glob import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.tools.build_log import EasyBuildError, print_msg diff --git a/easybuild/easyblocks/p/parmetis.py b/easybuild/easyblocks/p/parmetis.py index a9885b1c75..807ba8bd53 100644 --- a/easybuild/easyblocks/p/parmetis.py +++ b/easybuild/easyblocks/p/parmetis.py @@ -34,7 +34,7 @@ """ import os import shutil -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.framework.easyblock import EasyBlock from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/p/perl.py b/easybuild/easyblocks/p/perl.py index 11542473cf..93cc07d6ef 100644 --- a/easybuild/easyblocks/p/perl.py +++ b/easybuild/easyblocks/p/perl.py @@ -28,7 +28,7 @@ @author: Jens Timmerman (Ghent University) @author: Kenneth Hoste (Ghent University) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import glob import os import stat diff --git a/easybuild/easyblocks/p/petsc.py b/easybuild/easyblocks/p/petsc.py index 23c18c169c..a658dd0bbf 100644 --- a/easybuild/easyblocks/p/petsc.py +++ b/easybuild/easyblocks/p/petsc.py @@ -29,7 +29,7 @@ """ import os import re -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain diff --git a/easybuild/easyblocks/p/pgi.py b/easybuild/easyblocks/p/pgi.py index 8d34ae7b06..38e8076756 100644 --- a/easybuild/easyblocks/p/pgi.py +++ b/easybuild/easyblocks/p/pgi.py @@ -39,7 +39,7 @@ import sys import easybuild.tools.environment as env -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.packedbinary import PackedBinary from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.easyconfig.types import ensure_iterable_license_specs diff --git a/easybuild/easyblocks/p/picard.py b/easybuild/easyblocks/p/picard.py index b9f2d1a9b1..e49bc58ca0 100644 --- a/easybuild/easyblocks/p/picard.py +++ b/easybuild/easyblocks/p/picard.py @@ -35,7 +35,7 @@ import os import re import shutil -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.tarball import Tarball from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/p/psi.py b/easybuild/easyblocks/p/psi.py index af400c3cec..14324a2711 100644 --- a/easybuild/easyblocks/p/psi.py +++ b/easybuild/easyblocks/p/psi.py @@ -29,7 +29,7 @@ @author: Ward Poelmans (Ghent University) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import glob import os import shutil diff --git a/easybuild/easyblocks/p/psmpi.py b/easybuild/easyblocks/p/psmpi.py index 755cc1f809..f12b6aa5d9 100644 --- a/easybuild/easyblocks/p/psmpi.py +++ b/easybuild/easyblocks/p/psmpi.py @@ -30,7 +30,7 @@ import easybuild.tools.toolchain as toolchain -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.mpich import EB_MPICH from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/p/python.py b/easybuild/easyblocks/p/python.py index 975935cdfb..d01cf50ae8 100644 --- a/easybuild/easyblocks/p/python.py +++ b/easybuild/easyblocks/p/python.py @@ -38,7 +38,7 @@ import fileinput import sys import tempfile -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env from easybuild.easyblocks.generic.configuremake import ConfigureMake diff --git a/easybuild/easyblocks/p/pytorch.py b/easybuild/easyblocks/p/pytorch.py index f40cf23f92..ec9b40a5ee 100644 --- a/easybuild/easyblocks/p/pytorch.py +++ b/easybuild/easyblocks/p/pytorch.py @@ -32,7 +32,7 @@ import re import tempfile import easybuild.tools.environment as env -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.pythonpackage import PythonPackage from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError, print_warning diff --git a/easybuild/easyblocks/q/qscintilla.py b/easybuild/easyblocks/q/qscintilla.py index 3a64d6c607..ef98de5df6 100644 --- a/easybuild/easyblocks/q/qscintilla.py +++ b/easybuild/easyblocks/q/qscintilla.py @@ -29,7 +29,7 @@ author: Maxime Boissonneault (Compute Canada) """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.easyblocks.generic.pythonpackage import det_pylibdir diff --git a/easybuild/easyblocks/q/qt.py b/easybuild/easyblocks/q/qt.py index cc23f27a99..2cace72e0c 100644 --- a/easybuild/easyblocks/q/qt.py +++ b/easybuild/easyblocks/q/qt.py @@ -28,7 +28,7 @@ @author: Kenneth Hoste (Ghent University) """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain diff --git a/easybuild/easyblocks/q/quantumespresso.py b/easybuild/easyblocks/q/quantumespresso.py index b5a232817b..15d00f53d0 100644 --- a/easybuild/easyblocks/q/quantumespresso.py +++ b/easybuild/easyblocks/q/quantumespresso.py @@ -33,7 +33,7 @@ import re import shutil import sys -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain diff --git a/easybuild/easyblocks/r/r.py b/easybuild/easyblocks/r/r.py index 5f73952873..27d5d2ac32 100644 --- a/easybuild/easyblocks/r/r.py +++ b/easybuild/easyblocks/r/r.py @@ -30,7 +30,7 @@ """ import os import re -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env from easybuild.easyblocks.generic.configuremake import ConfigureMake diff --git a/easybuild/easyblocks/r/repeatmasker.py b/easybuild/easyblocks/r/repeatmasker.py index 0a4d10329b..e0a50b1a1d 100644 --- a/easybuild/easyblocks/r/repeatmasker.py +++ b/easybuild/easyblocks/r/repeatmasker.py @@ -25,7 +25,7 @@ """ EasyBuild support for building and installing RepeatMasker, implemented as an easyblock """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import os from easybuild.easyblocks.generic.tarball import Tarball diff --git a/easybuild/easyblocks/r/rmpi.py b/easybuild/easyblocks/r/rmpi.py index c9a1528acd..8bca1893be 100644 --- a/easybuild/easyblocks/r/rmpi.py +++ b/easybuild/easyblocks/r/rmpi.py @@ -33,7 +33,7 @@ @author: Balazs Hajgato (Vrije Universiteit Brussel) """ import easybuild.tools.toolchain as toolchain -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.rpackage import RPackage diff --git a/easybuild/easyblocks/r/root.py b/easybuild/easyblocks/r/root.py index 872671eb4b..2a10a86d57 100644 --- a/easybuild/easyblocks/r/root.py +++ b/easybuild/easyblocks/r/root.py @@ -28,7 +28,7 @@ @author: Kenneth Hoste (Ghent University) @author: Jens Timmerman (Ghent University) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import os from easybuild.framework.easyconfig import CUSTOM diff --git a/easybuild/easyblocks/s/samtools.py b/easybuild/easyblocks/s/samtools.py index 6d297c2283..4be92850f0 100644 --- a/easybuild/easyblocks/s/samtools.py +++ b/easybuild/easyblocks/s/samtools.py @@ -19,7 +19,7 @@ @author: Fotis Georgatos (Uni.Lu) @author: Kenneth Hoste (Ghent University) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import glob import os import stat diff --git a/easybuild/easyblocks/s/scalapack.py b/easybuild/easyblocks/s/scalapack.py index 466fef57c3..11d285e4ae 100644 --- a/easybuild/easyblocks/s/scalapack.py +++ b/easybuild/easyblocks/s/scalapack.py @@ -34,7 +34,7 @@ import glob import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.blacs import det_interface # @UnresolvedImport diff --git a/easybuild/easyblocks/s/scalasca1.py b/easybuild/easyblocks/s/scalasca1.py index a78398cc9d..d991af9a31 100644 --- a/easybuild/easyblocks/s/scalasca1.py +++ b/easybuild/easyblocks/s/scalasca1.py @@ -29,7 +29,7 @@ @author: Bernd Mohr (Juelich Supercomputing Centre) """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.generic.configuremake import ConfigureMake diff --git a/easybuild/easyblocks/s/scipion.py b/easybuild/easyblocks/s/scipion.py index 86d7460a33..42d21a981b 100644 --- a/easybuild/easyblocks/s/scipion.py +++ b/easybuild/easyblocks/s/scipion.py @@ -30,7 +30,7 @@ """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.framework.extensioneasyblock import ExtensionEasyBlock from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import copy, mkdir, symlink diff --git a/easybuild/easyblocks/s/scipy.py b/easybuild/easyblocks/s/scipy.py index 094a0259b5..b0551a2b74 100644 --- a/easybuild/easyblocks/s/scipy.py +++ b/easybuild/easyblocks/s/scipy.py @@ -35,7 +35,7 @@ """ import os import tempfile -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain diff --git a/easybuild/easyblocks/s/scotch.py b/easybuild/easyblocks/s/scotch.py index 89c728f7c9..b09d3cdc0d 100644 --- a/easybuild/easyblocks/s/scotch.py +++ b/easybuild/easyblocks/s/scotch.py @@ -32,7 +32,7 @@ @author: Jens Timmerman (Ghent University) """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.toolchain as toolchain from easybuild.framework.easyblock import EasyBlock diff --git a/easybuild/easyblocks/s/siesta.py b/easybuild/easyblocks/s/siesta.py index 81b132236e..69a911fa80 100644 --- a/easybuild/easyblocks/s/siesta.py +++ b/easybuild/easyblocks/s/siesta.py @@ -32,7 +32,7 @@ import stat import easybuild.tools.toolchain as toolchain -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/s/slepc.py b/easybuild/easyblocks/s/slepc.py index 0254f195ad..43ecff7a4c 100644 --- a/easybuild/easyblocks/s/slepc.py +++ b/easybuild/easyblocks/s/slepc.py @@ -30,7 +30,7 @@ import os import re -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env from easybuild.easyblocks.generic.configuremake import ConfigureMake diff --git a/easybuild/easyblocks/s/snphylo.py b/easybuild/easyblocks/s/snphylo.py index 88eb26f7ae..7fed0ace86 100644 --- a/easybuild/easyblocks/s/snphylo.py +++ b/easybuild/easyblocks/s/snphylo.py @@ -31,7 +31,7 @@ import os import re import stat -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.framework.easyblock import EasyBlock from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/s/suitesparse.py b/easybuild/easyblocks/s/suitesparse.py index 7afd352080..a7f8f6d65f 100644 --- a/easybuild/easyblocks/s/suitesparse.py +++ b/easybuild/easyblocks/s/suitesparse.py @@ -36,7 +36,7 @@ import re import os import sys -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/s/superlu.py b/easybuild/easyblocks/s/superlu.py index eafa3c1e3e..9e85160587 100644 --- a/easybuild/easyblocks/s/superlu.py +++ b/easybuild/easyblocks/s/superlu.py @@ -30,7 +30,7 @@ """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.cmakemake import CMakeMake from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/t/tbb.py b/easybuild/easyblocks/t/tbb.py index e3f34e1ab8..995bfa854a 100644 --- a/easybuild/easyblocks/t/tbb.py +++ b/easybuild/easyblocks/t/tbb.py @@ -37,7 +37,7 @@ import glob import os import shutil -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.easyblocks.generic.intelbase import INSTALL_MODE_NAME_2015, INSTALL_MODE_2015 diff --git a/easybuild/easyblocks/t/tensorrt.py b/easybuild/easyblocks/t/tensorrt.py index 204302e7d6..8c818b959c 100644 --- a/easybuild/easyblocks/t/tensorrt.py +++ b/easybuild/easyblocks/t/tensorrt.py @@ -30,7 +30,7 @@ """ import glob import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.binary import Binary from easybuild.easyblocks.generic.pythonpackage import PythonPackage, PIP_INSTALL_CMD diff --git a/easybuild/easyblocks/t/tinker.py b/easybuild/easyblocks/t/tinker.py index 4892873b0d..78ecd57312 100644 --- a/easybuild/easyblocks/t/tinker.py +++ b/easybuild/easyblocks/t/tinker.py @@ -31,7 +31,7 @@ import os import tempfile -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.toolchain as toolchain from easybuild.framework.easyblock import EasyBlock diff --git a/easybuild/easyblocks/t/tkinter.py b/easybuild/easyblocks/t/tkinter.py index 8f4312f4b5..eb792c876c 100644 --- a/easybuild/easyblocks/t/tkinter.py +++ b/easybuild/easyblocks/t/tkinter.py @@ -33,7 +33,7 @@ import glob import os import tempfile -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env from easybuild.easyblocks.generic.pythonpackage import det_pylibdir diff --git a/easybuild/easyblocks/t/trilinos.py b/easybuild/easyblocks/t/trilinos.py index aa4b911473..030fbcd35b 100644 --- a/easybuild/easyblocks/t/trilinos.py +++ b/easybuild/easyblocks/t/trilinos.py @@ -31,7 +31,7 @@ import random import re -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.generic.cmakemake import CMakeMake diff --git a/easybuild/easyblocks/t/trinity.py b/easybuild/easyblocks/t/trinity.py index 7eeedf6e09..72cf9e454b 100644 --- a/easybuild/easyblocks/t/trinity.py +++ b/easybuild/easyblocks/t/trinity.py @@ -36,7 +36,7 @@ import glob import os import shutil -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.toolchain as toolchain from easybuild.framework.easyblock import EasyBlock diff --git a/easybuild/easyblocks/u/ufc.py b/easybuild/easyblocks/u/ufc.py index f77cf7ea24..361409dff7 100644 --- a/easybuild/easyblocks/u/ufc.py +++ b/easybuild/easyblocks/u/ufc.py @@ -27,7 +27,7 @@ @author: Kenneth Hoste (Ghent University) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.cmakepythonpackage import CMakePythonPackage from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/v/vmd.py b/easybuild/easyblocks/v/vmd.py index c7273ab5c6..3c122bd2e0 100644 --- a/easybuild/easyblocks/v/vmd.py +++ b/easybuild/easyblocks/v/vmd.py @@ -31,7 +31,7 @@ """ import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.easyblocks.generic.pythonpackage import det_pylibdir from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/v/vtune.py b/easybuild/easyblocks/v/vtune.py index 40253f57af..5459f38b00 100644 --- a/easybuild/easyblocks/v/vtune.py +++ b/easybuild/easyblocks/v/vtune.py @@ -28,7 +28,7 @@ @author: Kenneth Hoste (Ghent University) @author: Damian Alvarez (Forschungzentrum Juelich GmbH) """ -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import os from easybuild.easyblocks.generic.intelbase import IntelBase, ACTIVATION_NAME_2012, LICENSE_FILE_NAME_2012 diff --git a/easybuild/easyblocks/w/wien2k.py b/easybuild/easyblocks/w/wien2k.py index 07c32ad40c..32430c4f21 100644 --- a/easybuild/easyblocks/w/wien2k.py +++ b/easybuild/easyblocks/w/wien2k.py @@ -39,7 +39,7 @@ import shutil import sys import tempfile -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain diff --git a/easybuild/easyblocks/w/wps.py b/easybuild/easyblocks/w/wps.py index d6c6cca849..4dba12ac02 100644 --- a/easybuild/easyblocks/w/wps.py +++ b/easybuild/easyblocks/w/wps.py @@ -35,7 +35,7 @@ import os import re import tempfile -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain diff --git a/easybuild/easyblocks/w/wrf.py b/easybuild/easyblocks/w/wrf.py index ddd6387fd8..a21eabd12c 100644 --- a/easybuild/easyblocks/w/wrf.py +++ b/easybuild/easyblocks/w/wrf.py @@ -35,7 +35,7 @@ import os import re -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain diff --git a/easybuild/easyblocks/w/wxpython.py b/easybuild/easyblocks/w/wxpython.py index 939e2718a1..524b90a6a8 100644 --- a/easybuild/easyblocks/w/wxpython.py +++ b/easybuild/easyblocks/w/wxpython.py @@ -32,7 +32,7 @@ import glob import os -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.pythonpackage import PythonPackage, det_python_version from easybuild.tools.filetools import change_dir, symlink from easybuild.tools.modules import get_software_root diff --git a/easybuild/easyblocks/x/xmipp.py b/easybuild/easyblocks/x/xmipp.py index 95230cd03e..3043a4c93a 100644 --- a/easybuild/easyblocks/x/xmipp.py +++ b/easybuild/easyblocks/x/xmipp.py @@ -34,7 +34,7 @@ import os import easybuild.tools.environment as env -from distutils.version import LooseVersion +from easybuild.tools import LooseVersion from easybuild.easyblocks.generic.scons import SCons from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import apply_regex_substitutions, change_dir From c5b7f463db2a495b53e2ad0cca97b81ebd73d119 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 7 Dec 2023 14:26:57 +0000 Subject: [PATCH 079/145] Deprecate 'verbose' parameter for build_step in ConfigureMake --- easybuild/easyblocks/generic/configuremake.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/generic/configuremake.py b/easybuild/easyblocks/generic/configuremake.py index ec9f0527f3..39e8825b65 100644 --- a/easybuild/easyblocks/generic/configuremake.py +++ b/easybuild/easyblocks/generic/configuremake.py @@ -327,12 +327,15 @@ def configure_step(self, cmd_prefix=''): return res.output - def build_step(self, verbose=True, path=None): + def build_step(self, verbose=None, path=None): """ Start the actual build - typical: make -j X """ + if verbose is not None: + self.log.deprecated("The 'verbose' parameter to build_step is deprecated and unneeded.", '6.0') + paracmd = '' if self.cfg['parallel']: paracmd = "-j %s" % self.cfg['parallel'] @@ -351,7 +354,7 @@ def build_step(self, verbose=True, path=None): ]) self.log.info("Building target '%s'", target) - res = run_shell_cmd(cmd, work_dir=path, output_file=verbose) + res = run_shell_cmd(cmd, work_dir=path) out = res.output return out From d5e54a9f22620c81e249f205bdcf490800e8f20d Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Thu, 7 Dec 2023 15:02:51 +0000 Subject: [PATCH 080/145] Use fail_on_error=False in det_cmake_version(). It handles errors itself already by just looking at the output, no need also do it in run_shell_cmd here. This fixes the test case for det_cmake_version(). --- easybuild/easyblocks/generic/cmakemake.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/cmakemake.py b/easybuild/easyblocks/generic/cmakemake.py index 030783df05..6838a68168 100644 --- a/easybuild/easyblocks/generic/cmakemake.py +++ b/easybuild/easyblocks/generic/cmakemake.py @@ -63,7 +63,7 @@ def det_cmake_version(): regex = re.compile(r"^[cC][mM]ake version (?P[0-9]\.[0-9a-zA-Z.-]+)$", re.M) cmd = "cmake --version" - cmd_res = run_shell_cmd(cmd, hidden=True) + cmd_res = run_shell_cmd(cmd, hidden=True, fail_on_error=False) out = cmd_res.output res = regex.search(out) if res: From e0bfdabc1801a4abe4eff7192b41ed71e44e14c3 Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Fri, 8 Dec 2023 22:34:03 +0100 Subject: [PATCH 081/145] fix+enhance NWChem easyblock --- easybuild/easyblocks/n/nwchem.py | 54 ++++++++++++-------------------- 1 file changed, 20 insertions(+), 34 deletions(-) diff --git a/easybuild/easyblocks/n/nwchem.py b/easybuild/easyblocks/n/nwchem.py index 667b0bbd1e..3fa9c8cb33 100644 --- a/easybuild/easyblocks/n/nwchem.py +++ b/easybuild/easyblocks/n/nwchem.py @@ -29,7 +29,6 @@ """ import os import re -import shutil import stat import tempfile @@ -40,7 +39,8 @@ from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import adjust_permissions, change_dir, mkdir, remove_file, symlink, write_file +from easybuild.tools.filetools import (adjust_permissions, change_dir, copy_dir, copy_file, mkdir, remove_file, + remove_dir, symlink, write_file) from easybuild.tools.modules import get_software_libdir, get_software_root, get_software_version from easybuild.tools.run import run_cmd @@ -325,16 +325,19 @@ def install_step(self): # binary bindir = os.path.join(self.installdir, 'bin') mkdir(bindir) - shutil.copy(os.path.join(self.cfg['start_dir'], 'bin', self.cfg['target'], 'nwchem'), - bindir) + copy_file(os.path.join(self.cfg['start_dir'], 'bin', self.cfg['target'], 'nwchem'), + os.path.join(bindir, 'nwchem')) # data - shutil.copytree(os.path.join(self.cfg['start_dir'], 'src', 'data'), - os.path.join(self.installdir, 'data')) - shutil.copytree(os.path.join(self.cfg['start_dir'], 'src', 'basis', 'libraries'), - os.path.join(self.installdir, 'data', 'libraries')) - shutil.copytree(os.path.join(self.cfg['start_dir'], 'src', 'nwpw', 'libraryps'), - os.path.join(self.installdir, 'data', 'libraryps')) + copy_dir(os.path.join(self.cfg['start_dir'], 'src', 'data'), + os.path.join(self.installdir, 'data')) + copy_dir(os.path.join(self.cfg['start_dir'], 'src', 'basis', 'libraries'), + os.path.join(self.installdir, 'data', 'libraries')) + copy_dir(os.path.join(self.cfg['start_dir'], 'src', 'nwpw', 'libraryps'), + os.path.join(self.installdir, 'data', 'libraryps')) + # examples (needed for the test_cases_step) + copy_dir(os.path.join(self.cfg['start_dir'], 'examples'), + os.path.join(self.installdir, 'examples')) except OSError as err: raise EasyBuildError("Failed to install NWChem: %s", err) @@ -388,23 +391,6 @@ def make_module_extra(self): return txt - def cleanup_step(self): - """Copy stuff from build directory we still need, if any.""" - - try: - exs_dir = os.path.join(self.cfg['start_dir'], 'examples') - - self.examples_dir = os.path.join(tempfile.mkdtemp(), 'examples') - - shutil.copytree(exs_dir, self.examples_dir) - - self.log.info("Copied %s to %s." % (exs_dir, self.examples_dir)) - - except OSError as err: - raise EasyBuildError("Failed to copy examples: %s", err) - - super(EB_NWChem, self).cleanup_step() - def test_cases_step(self): """Run provided list of test cases, or provided examples is no test cases were specified.""" @@ -438,7 +424,7 @@ def test_cases_step(self): ('md/benzene', ['benzene.nw']) ] - self.cfg['tests'] = [(os.path.join(self.examples_dir, d), l) for (d, l) in examples] + self.cfg['tests'] = [(os.path.join(self.installdir, 'examples', d), l) for (d, l) in examples] self.log.info("List of examples to be run as test cases: %s" % self.cfg['tests']) try: @@ -452,7 +438,7 @@ def test_cases_step(self): local_nwchemrc_dir = os.path.dirname(self.local_nwchemrc) if not os.path.exists(local_nwchemrc_dir): os.makedirs(local_nwchemrc_dir) - shutil.copy2(default_nwchemrc, self.local_nwchemrc) + copy_file(default_nwchemrc, self.local_nwchemrc) # only try to create symlink if it's not there yet # we've verified earlier that the symlink is what we expect it to be if it's there @@ -483,11 +469,12 @@ def test_cases_step(self): test_file = os.path.join(testdir, item) if os.path.isfile(test_file): self.log.debug("Copying %s to %s" % (test_file, tmpdir)) - shutil.copy2(test_file, tmpdir) + copy_file(test_file, tmpdir) # run tests + n_mpi_ranks = min(self.cfg['parallel'], 4) for testx in tests: - cmd = "nwchem %s" % testx + cmd = '%s %s' % (self.toolchain.mpi_cmd_for('nwchem', n_mpi_ranks), testx) msg = "Running test '%s' (from %s) in %s..." % (cmd, testdir, tmpdir) self.log.info(msg) test_cases_log.write("\n%s\n" % msg) @@ -516,7 +503,7 @@ def test_cases_step(self): # go back change_dir(cwd) - shutil.rmtree(tmpdir) + remove_dir(tmpdir) fail_ratio = fail / tot fail_pcnt = fail_ratio * 100 @@ -534,8 +521,7 @@ def test_cases_step(self): # cleanup try: - shutil.rmtree(self.examples_dir) - shutil.rmtree(local_nwchemrc_dir) + remove_dir(local_nwchemrc_dir) except OSError as err: raise EasyBuildError("Cleanup failed: %s", err) From 440f9e479b17354c74c258463ab701e66afe0b21 Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Sun, 10 Dec 2023 19:08:34 +0100 Subject: [PATCH 082/145] run band.nw with 1 rank; set parallel --- easybuild/easyblocks/n/nwchem.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/n/nwchem.py b/easybuild/easyblocks/n/nwchem.py index 3fa9c8cb33..dc69fedca4 100644 --- a/easybuild/easyblocks/n/nwchem.py +++ b/easybuild/easyblocks/n/nwchem.py @@ -458,6 +458,10 @@ def test_cases_step(self): test_cases_logfn = os.path.join(self.installdir, config.log_path(), 'test_cases.log') test_cases_log = open(test_cases_logfn, "w") + # ensure parallel is set in case check_readiness_step is skipped (e.g. with --sanity-check-only) + if not self.cfg['parallel']: + self.set_parallel() + for (testdir, tests) in self.cfg['tests']: # run test in a temporary dir @@ -472,8 +476,11 @@ def test_cases_step(self): copy_file(test_file, tmpdir) # run tests - n_mpi_ranks = min(self.cfg['parallel'], 4) for testx in tests: + if testx == 'band.nw' and LooseVersion(self.version) < LooseVersion("7.2"): + n_mpi_ranks = 1 + else: + n_mpi_ranks = min(self.cfg['parallel'], 4) cmd = '%s %s' % (self.toolchain.mpi_cmd_for('nwchem', n_mpi_ranks), testx) msg = "Running test '%s' (from %s) in %s..." % (cmd, testdir, tmpdir) self.log.info(msg) From 0b8bee4876820c6a75ea37d03dabc8b1e4cb8a2f Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Sun, 10 Dec 2023 19:11:33 +0100 Subject: [PATCH 083/145] slight reorder --- easybuild/easyblocks/n/nwchem.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/n/nwchem.py b/easybuild/easyblocks/n/nwchem.py index dc69fedca4..50c04a3a88 100644 --- a/easybuild/easyblocks/n/nwchem.py +++ b/easybuild/easyblocks/n/nwchem.py @@ -475,12 +475,13 @@ def test_cases_step(self): self.log.debug("Copying %s to %s" % (test_file, tmpdir)) copy_file(test_file, tmpdir) + max_mpi_ranks = min(self.cfg['parallel'], 4) # run tests for testx in tests: if testx == 'band.nw' and LooseVersion(self.version) < LooseVersion("7.2"): n_mpi_ranks = 1 else: - n_mpi_ranks = min(self.cfg['parallel'], 4) + n_mpi_ranks = max_mpi_ranks cmd = '%s %s' % (self.toolchain.mpi_cmd_for('nwchem', n_mpi_ranks), testx) msg = "Running test '%s' (from %s) in %s..." % (cmd, testdir, tmpdir) self.log.info(msg) From b7d780e051ef6a77d31792aeb71875e42969ca42 Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Sun, 10 Dec 2023 19:13:21 +0100 Subject: [PATCH 084/145] update comment --- easybuild/easyblocks/n/nwchem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/n/nwchem.py b/easybuild/easyblocks/n/nwchem.py index 50c04a3a88..a9495ccbe9 100644 --- a/easybuild/easyblocks/n/nwchem.py +++ b/easybuild/easyblocks/n/nwchem.py @@ -458,7 +458,7 @@ def test_cases_step(self): test_cases_logfn = os.path.join(self.installdir, config.log_path(), 'test_cases.log') test_cases_log = open(test_cases_logfn, "w") - # ensure parallel is set in case check_readiness_step is skipped (e.g. with --sanity-check-only) + # ensure parallel is set in case check_readiness_step is skipped if not self.cfg['parallel']: self.set_parallel() From d553f228e296cb42b673d32e7d2bf60204c4507b Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Mon, 11 Dec 2023 11:16:53 +0100 Subject: [PATCH 085/145] always set MPI ranks to 1 --- easybuild/easyblocks/n/nwchem.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/easybuild/easyblocks/n/nwchem.py b/easybuild/easyblocks/n/nwchem.py index a9495ccbe9..6eac2b6e68 100644 --- a/easybuild/easyblocks/n/nwchem.py +++ b/easybuild/easyblocks/n/nwchem.py @@ -458,10 +458,6 @@ def test_cases_step(self): test_cases_logfn = os.path.join(self.installdir, config.log_path(), 'test_cases.log') test_cases_log = open(test_cases_logfn, "w") - # ensure parallel is set in case check_readiness_step is skipped - if not self.cfg['parallel']: - self.set_parallel() - for (testdir, tests) in self.cfg['tests']: # run test in a temporary dir @@ -478,10 +474,8 @@ def test_cases_step(self): max_mpi_ranks = min(self.cfg['parallel'], 4) # run tests for testx in tests: - if testx == 'band.nw' and LooseVersion(self.version) < LooseVersion("7.2"): - n_mpi_ranks = 1 - else: - n_mpi_ranks = max_mpi_ranks + # some test cases hang with more than 1 rank on some architectures + n_mpi_ranks = 1 cmd = '%s %s' % (self.toolchain.mpi_cmd_for('nwchem', n_mpi_ranks), testx) msg = "Running test '%s' (from %s) in %s..." % (cmd, testdir, tmpdir) self.log.info(msg) From c456d1007fddfe0d7812a97fb6f96c6a6632694e Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Mon, 11 Dec 2023 11:18:01 +0100 Subject: [PATCH 086/145] remove unused var --- easybuild/easyblocks/n/nwchem.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/easybuild/easyblocks/n/nwchem.py b/easybuild/easyblocks/n/nwchem.py index 6eac2b6e68..3c5bf1e539 100644 --- a/easybuild/easyblocks/n/nwchem.py +++ b/easybuild/easyblocks/n/nwchem.py @@ -471,10 +471,9 @@ def test_cases_step(self): self.log.debug("Copying %s to %s" % (test_file, tmpdir)) copy_file(test_file, tmpdir) - max_mpi_ranks = min(self.cfg['parallel'], 4) # run tests for testx in tests: - # some test cases hang with more than 1 rank on some architectures + # some test cases hang with more than 1 MPI rank on some architectures n_mpi_ranks = 1 cmd = '%s %s' % (self.toolchain.mpi_cmd_for('nwchem', n_mpi_ranks), testx) msg = "Running test '%s' (from %s) in %s..." % (cmd, testdir, tmpdir) From 1d53af364f14dbb24a28bc1b79efd6713ae851bc Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Tue, 12 Dec 2023 11:28:25 +0000 Subject: [PATCH 087/145] adding easyblocks: dualsphysics.py --- easybuild/easyblocks/d/dualsphysics.py | 151 +++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 easybuild/easyblocks/d/dualsphysics.py diff --git a/easybuild/easyblocks/d/dualsphysics.py b/easybuild/easyblocks/d/dualsphysics.py new file mode 100644 index 0000000000..5d405b3a68 --- /dev/null +++ b/easybuild/easyblocks/d/dualsphysics.py @@ -0,0 +1,151 @@ +## +# Copyright 2013-2023 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for building and installing DualSPHysics, implemented as an easyblock + +@author: Jasper Grimm (University of York) +""" +import glob +import os +import stat + +from easybuild.easyblocks.generic.cmakemakecp import CMakeMakeCp +from easybuild.framework.easyconfig import CUSTOM +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.config import build_option +from easybuild.tools.filetools import adjust_permissions +from easybuild.tools.modules import get_software_root +from easybuild.tools.run import run_cmd + +class EB_DualSPHysics(CMakeMakeCp): + """Support for building/installing DualSPHysics.""" + + @staticmethod + def extra_options(): + """Extra easyconfig parameters for DualSPHysics.""" + extra_vars = CMakeMakeCp.extra_options() + + extra_vars['separate_build_dir'][0] = True + + # files_to_copy is not mandatory here since we set it in the easyblock + extra_vars['files_to_copy'][2] = CUSTOM + return extra_vars + + def __init__(self, *args, **kwargs): + """Initialize calss variables custom to DualSPHysics.""" + super(EB_DualSPHysics, self).__init__(*args, **kwargs) + + self.dsph_target = None + self.shortver = None + + def prepare_step(self): + """Determine name of binary that will be installed.""" + super(EB_DualSPHysics, self).prepare_step() + + if get_software_root('CUDA'): + self.dsph_target = 'GPU' + else: + self.dsph_target = 'CPU' + + self.shortver = '.'.join(self.version.split('.')[0:2]) + + def configure_step(self): + """Custom configure procedure for DualSPHysics.""" + srcdir = os.path.join(self.cfg['start_dir'], 'src/source') + CMakeMakeCp.configure_step(self, srcdir=srcdir) + + def install_step(self): + """Custom install procedure for DualSPHysics.""" + # *_linux64 binaries are missing execute permissions + bindir = os.path.join(self.cfg['start_dir'], 'bin', 'linux') + for b in glob.glob(os.path.join(bindir, '*_linux64')): + perms = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH + adjust_permissions(b, perms, add=True) + + # no `make install` target + self.cfg['files_to_copy'] = [ + (['bin/linux/*'], 'bin'), + (['src/lib/linux_gcc/*'], 'lib'), + ] + super(EB_DualSPHysics, self).install_step() + + def post_install_step(self): + """Custom post-installation step: ensure rpath is patched into binaries/libraries if configured.""" + super(EB_DualSPHysics, self).post_install_step() + + if build_option('rpath'): + # only the compiled binary (e.g. DualSPHysics5.0CPU_linux64) is rpath'd, the precompiled libraries + # and binaries are not + # simple solution: copy the RPATH from the compiled binary to the others, then strip excess paths + rpathed_bin = os.path.join( + self.installdir, 'bin', 'DualSPHysics%s%s_linux64' % (self.shortver, self.dsph_target) + ) + + out, _ = run_cmd("patchelf --print-rpath %s" % rpathed_bin, simple=False, trace=False) + comp_rpath = out.strip() + + files_to_patch = [] + for x in [('bin', '*_linux64'), ('bin', '*.so'), ('lib', '*.so')]: + files_to_patch.extend(glob.glob(os.path.join(self.installdir, *x))) + + try: + for x in files_to_patch: + out, _ = run_cmd("patchelf --print-rpath %s" % x, trace=False) + self.log.debug("Original RPATH for %s: %s" % (out, x)) + + run_cmd("patchelf --set-rpath '%s' --force-rpath %s" % (comp_rpath, x), trace=False) + run_cmd("patchelf --shrink-rpath --force-rpath %s" % x, trace=False) + + out, _ = run_cmd("patchelf --print-rpath %s" % x, trace=False) + self.log.debug("RPATH for %s (after patching and shrinking): %s" % (out, x)) + + except OSError as err: + raise EasyBuildError("Failed to patch RPATH section in binaries/libraries: %s", err) + + + def sanity_check_step(self): + """Custom sanity checks for DualSPHysics.""" + + # repeated here in case other steps are skipped (e.g. due to --sanity-check-only) + if get_software_root('CUDA'): + self.dsph_target = 'GPU' + else: + self.dsph_target = 'CPU' + + self.shortver = '.'.join(self.version.split('.')[0:2]) + + bins = ['GenCase', 'PartVTK', 'IsoSurface', 'MeasureTool', 'GenCase_MkWord', 'DualSPHysics4.0_LiquidGas', + 'DualSPHysics4.0_LiquidGasCPU', 'DualSPHysics%s' % self.shortver, + 'DualSPHysics%s%s' % (self.shortver, self.dsph_target), 'DualSPHysics%s_NNewtonian' % self.shortver, + 'DualSPHysics%s_NNewtonianCPU' % self.shortver] + + custom_paths = { + 'files': ['bin/%s_linux64' % x for x in bins], + 'dirs': ['lib'], + } + + custom_commands = ['%s_linux64 -h' % x for x in bins] + + super(EB_DualSPHysics, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) From 6e688d43d19ea568e23257ae256b137409c5ad22 Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Tue, 12 Dec 2023 11:31:04 +0000 Subject: [PATCH 088/145] hound fixes --- easybuild/easyblocks/d/dualsphysics.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/easybuild/easyblocks/d/dualsphysics.py b/easybuild/easyblocks/d/dualsphysics.py index 5d405b3a68..33ccb3064a 100644 --- a/easybuild/easyblocks/d/dualsphysics.py +++ b/easybuild/easyblocks/d/dualsphysics.py @@ -39,6 +39,7 @@ from easybuild.tools.modules import get_software_root from easybuild.tools.run import run_cmd + class EB_DualSPHysics(CMakeMakeCp): """Support for building/installing DualSPHysics.""" @@ -48,7 +49,7 @@ def extra_options(): extra_vars = CMakeMakeCp.extra_options() extra_vars['separate_build_dir'][0] = True - + # files_to_copy is not mandatory here since we set it in the easyblock extra_vars['files_to_copy'][2] = CUSTOM return extra_vars @@ -59,11 +60,11 @@ def __init__(self, *args, **kwargs): self.dsph_target = None self.shortver = None - + def prepare_step(self): """Determine name of binary that will be installed.""" super(EB_DualSPHysics, self).prepare_step() - + if get_software_root('CUDA'): self.dsph_target = 'GPU' else: @@ -124,10 +125,9 @@ def post_install_step(self): except OSError as err: raise EasyBuildError("Failed to patch RPATH section in binaries/libraries: %s", err) - def sanity_check_step(self): """Custom sanity checks for DualSPHysics.""" - + # repeated here in case other steps are skipped (e.g. due to --sanity-check-only) if get_software_root('CUDA'): self.dsph_target = 'GPU' @@ -140,7 +140,7 @@ def sanity_check_step(self): 'DualSPHysics4.0_LiquidGasCPU', 'DualSPHysics%s' % self.shortver, 'DualSPHysics%s%s' % (self.shortver, self.dsph_target), 'DualSPHysics%s_NNewtonian' % self.shortver, 'DualSPHysics%s_NNewtonianCPU' % self.shortver] - + custom_paths = { 'files': ['bin/%s_linux64' % x for x in bins], 'dirs': ['lib'], From 69587053dc3517c6629283c961286e0cfbac10bf Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Tue, 12 Dec 2023 11:38:10 +0000 Subject: [PATCH 089/145] fix prepare step --- easybuild/easyblocks/d/dualsphysics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/d/dualsphysics.py b/easybuild/easyblocks/d/dualsphysics.py index 33ccb3064a..f83bd010a4 100644 --- a/easybuild/easyblocks/d/dualsphysics.py +++ b/easybuild/easyblocks/d/dualsphysics.py @@ -61,9 +61,9 @@ def __init__(self, *args, **kwargs): self.dsph_target = None self.shortver = None - def prepare_step(self): + def prepare_step(self, *args, **kwargs): """Determine name of binary that will be installed.""" - super(EB_DualSPHysics, self).prepare_step() + super(EB_DualSPHysics, self).prepare_step(*args, **kwargs) if get_software_root('CUDA'): self.dsph_target = 'GPU' From a9be397361bcd39e903449a0fc83e72a0cc98a79 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Tue, 12 Dec 2023 13:07:46 +0000 Subject: [PATCH 090/145] Convert PythonPackage easyblock to run_shell_cmd. --- easybuild/easyblocks/generic/pythonpackage.py | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 76f94c1bc1..e65c712c3b 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -51,7 +51,7 @@ from easybuild.tools.config import build_option from easybuild.tools.filetools import change_dir, mkdir, remove_dir, symlink, which from easybuild.tools.modules import get_software_root -from easybuild.tools.run import run_cmd, subprocess_popen_text +from easybuild.tools.run import run_shell_cmd, subprocess_popen_text from easybuild.tools.utilities import nub from easybuild.tools.hooks import CONFIGURE_STEP, BUILD_STEP, TEST_STEP, INSTALL_STEP @@ -78,8 +78,8 @@ def det_python_version(python_cmd): """Determine version of specified 'python' command.""" pycode = 'import sys; print("%s.%s.%s" % sys.version_info[:3])' - out, _ = run_cmd("%s -c '%s'" % (python_cmd, pycode), simple=False, trace=False) - return out.strip() + res = run_shell_cmd("%s -c '%s'" % (python_cmd, pycode), hidden=True) + return res.output.strip() def pick_python_cmd(req_maj_ver=None, req_min_ver=None): @@ -169,7 +169,7 @@ def det_pylibdir(plat_specific=False, python_cmd=None): python_cmd = 'python' # determine Python lib dir via distutils - # use run_cmd, we can to talk to the active Python, not the system Python running EasyBuild + # use run_shell_cmd, we can to talk to the active Python, not the system Python running EasyBuild prefix = '/tmp/' args = 'plat_specific=%s, prefix="%s"' % (plat_specific, prefix) pycode = "import distutils.sysconfig; print(distutils.sysconfig.get_python_lib(%s))" % args @@ -177,13 +177,13 @@ def det_pylibdir(plat_specific=False, python_cmd=None): log.debug("Determining Python library directory using command '%s'", cmd) - out, ec = run_cmd(cmd, simple=False, force_in_dry_run=True, trace=False) - txt = out.strip().split('\n')[-1] + res = run_shell_cmd(cmd, in_dry_run=True, hidden=True) + txt = res.output.strip().split('\n')[-1] # value obtained should start with specified prefix, otherwise something is very wrong if not txt.startswith(prefix): raise EasyBuildError("Last line of output of %s does not start with specified prefix %s: %s (exit code %s)", - cmd, prefix, out, ec) + cmd, prefix, res.output, res.exit_code) pylibdir = txt[len(prefix):] log.debug("Determined pylibdir using '%s': %s", cmd, pylibdir) @@ -221,7 +221,8 @@ def det_pip_version(python_cmd='python'): log = fancylogger.getLogger('det_pip_version', fname=False) log.info("Determining pip version...") - out, _ = run_cmd("%s -m pip --version" % python_cmd, verbose=False, simple=False, trace=False) + res = run_shell_cmd("%s -m pip --version" % python_cmd, hidden=True) + out = res.output pip_version_regex = re.compile('^pip ([0-9.]+)') res = pip_version_regex.search(out) @@ -255,8 +256,8 @@ def det_py_install_scheme(python_cmd='python'): cmd = "%s -c 'import sysconfig; print(sysconfig.%s())'" % (python_cmd, get_default_scheme) log.debug("Determining active Python installation scheme with: %s", cmd) - out, _ = run_cmd(cmd, verbose=False, simple=False, trace=False) - py_install_scheme = out.strip() + res = run_shell_cmd(cmd, hidden=True) + py_install_scheme = res.output.strip() if py_install_scheme in PY_INSTALL_SCHEMES: log.info("Active Python installation scheme: %s", py_install_scheme) @@ -748,7 +749,7 @@ def configure_step(self): # creates log entries for python being used, for debugging cmd = "%(python)s -V; %(python)s -c 'import sys; print(sys.executable, sys.path)'" - run_cmd(cmd % {'python': self.python_cmd}, verbose=False, trace=False) + run_shell_cmd(cmd % {'python': self.python_cmd}, hidden=True) def build_step(self): """Build Python package using setup.py""" @@ -768,12 +769,12 @@ def build_step(self): if build_cmd: build_cmd = build_cmd % {'python': self.python_cmd} cmd = ' '.join([self.cfg['prebuildopts'], build_cmd, self.cfg['buildopts']]) - (out, _) = run_cmd(cmd, log_all=True, simple=False) + res = run_shell_cmd(cmd) # keep track of all output, so we can check for auto-downloaded dependencies; # take into account that build/install steps may be run multiple times # We consider the build and install output together as downloads likely happen here if this is run - self.install_cmd_output += out + self.install_cmd_output += res.output def test_step(self, return_output_ec=False): """ @@ -811,13 +812,13 @@ def test_step(self, return_output_ec=False): raise EasyBuildError("Failed to create test install dir: %s", err) # print Python search path (just debugging purposes) - run_cmd("%s -c 'import sys; print(sys.path)'" % self.python_cmd, verbose=False, trace=False) + run_shell_cmd("%s -c 'import sys; print(sys.path)'" % self.python_cmd, hidden=True) abs_pylibdirs = [os.path.join(actual_installdir, pylibdir) for pylibdir in self.all_pylibdirs] extrapath = "export PYTHONPATH=%s &&" % os.pathsep.join(abs_pylibdirs + ['$PYTHONPATH']) cmd = self.compose_install_command(test_installdir, extrapath=extrapath) - run_cmd(cmd, log_all=True, simple=True, verbose=False) + run_shell_cmd(cmd, hidden=True) self.py_post_install_shenanigans(test_installdir) @@ -830,12 +831,10 @@ def test_step(self, return_output_ec=False): self.cfg['testopts'], ]) + res = run_shell_cmd(cmd) if return_output_ec: - (out, ec) = run_cmd(cmd, log_all=False, log_ok=False, simple=False, regexp=False) - # need to log seperately, since log_all and log_ok need to be false to retrieve out and ec + (out, ec) = (res.output, res.exit_code) self.log.info("cmd '%s' exited with exit code %s and output:\n%s", cmd, ec, out) - else: - run_cmd(cmd, log_all=True, simple=True) if test_installdir: remove_dir(test_installdir) @@ -873,12 +872,12 @@ def install_step(self): # actually install Python package cmd = self.compose_install_command(self.installdir) - (out, _) = run_cmd(cmd, log_all=True, log_ok=True, simple=False) + res = run_shell_cmd(cmd) # keep track of all output from install command, so we can check for auto-downloaded dependencies; # take into account that install step may be run multiple times # (for iterated installations over multiply Python versions) - self.install_cmd_output += out + self.install_cmd_output += res.output self.py_post_install_shenanigans(self.installdir) @@ -1023,8 +1022,9 @@ def sanity_check_step(self, *args, **kwargs): pip_check_errors = [] - pip_check_msg, ec = run_cmd(pip_check_command, log_ok=False) - if ec: + res = run_shell_cmd(pip_check_command) + pip_check_msg = res.output + if res.exit_code: pip_check_errors.append('`%s` failed:\n%s' % (pip_check_command, pip_check_msg)) else: self.log.info('`%s` completed successfully' % pip_check_command) From d7b1d4644d3a662a92dc5f185b906b53d87a39a2 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Tue, 12 Dec 2023 13:34:09 +0000 Subject: [PATCH 091/145] Convert ConfigureMakePythonPackage easyblock to run_shell_cmd --- easybuild/easyblocks/generic/configuremakepythonpackage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/generic/configuremakepythonpackage.py b/easybuild/easyblocks/generic/configuremakepythonpackage.py index 8da38021a2..d9ebf8dc1c 100644 --- a/easybuild/easyblocks/generic/configuremakepythonpackage.py +++ b/easybuild/easyblocks/generic/configuremakepythonpackage.py @@ -31,7 +31,7 @@ """ from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.easyblocks.generic.pythonpackage import PythonPackage -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class ConfigureMakePythonPackage(ConfigureMake, PythonPackage): @@ -58,7 +58,7 @@ def configure_step(self, *args, **kwargs): """Configure build using ``python configure``.""" PythonPackage.configure_step(self, *args, **kwargs) cmd = ' '.join([self.cfg['preconfigopts'], self.python_cmd, self.cfg['configopts']]) - run_cmd(cmd, log_all=True) + run_shell_cmd(cmd) def build_step(self, *args, **kwargs): """Build Python package with ``make``.""" From 5d80cd84f60c9002cb7d760c11a4d7f0a0021a36 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Tue, 12 Dec 2023 13:35:21 +0000 Subject: [PATCH 092/145] Convert FortranPythonPackage easyblock to run_shell_cmd --- easybuild/easyblocks/generic/fortranpythonpackage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/generic/fortranpythonpackage.py b/easybuild/easyblocks/generic/fortranpythonpackage.py index 849dacf54b..1e434c50ca 100644 --- a/easybuild/easyblocks/generic/fortranpythonpackage.py +++ b/easybuild/easyblocks/generic/fortranpythonpackage.py @@ -36,7 +36,7 @@ import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.generic.pythonpackage import PythonPackage from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class FortranPythonPackage(PythonPackage): @@ -70,4 +70,4 @@ def build_step(self): raise EasyBuildError("Unknown family of compilers being used: %s", comp_fam) cmd = "%s %s setup.py build %s" % (self.cfg['prebuildopts'], self.python_cmd, self.cfg['buildopts']) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) From 53ef06778328392a3ab9d704dcfad5b59b8bd675 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Tue, 12 Dec 2023 13:35:46 +0000 Subject: [PATCH 093/145] Convert VersionIndependentPythonPackage easyblock to run_shell_cmd --- .../easyblocks/generic/versionindependentpythonpackage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/generic/versionindependentpythonpackage.py b/easybuild/easyblocks/generic/versionindependentpythonpackage.py index f6cc181b41..6c45efe7a8 100644 --- a/easybuild/easyblocks/generic/versionindependentpythonpackage.py +++ b/easybuild/easyblocks/generic/versionindependentpythonpackage.py @@ -37,7 +37,7 @@ import easybuild.tools.environment as env from easybuild.easyblocks.generic.pythonpackage import PythonPackage from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class VersionIndependentPythonPackage(PythonPackage): @@ -79,7 +79,7 @@ def install_step(self): raise EasyBuildError("%s easyblock is not compatible with using easy_install or pip", eb_name) cmd = self.compose_install_command(self.installdir) - run_cmd(cmd, log_all=True, simple=True, log_output=True) + run_shell_cmd(cmd) # setuptools stubbornly replaces the shebang line in scripts with # the full path to the Python interpreter used to install; From 3b31e92b3ab53b9339a2ea9e140cf8508ef45926 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Tue, 12 Dec 2023 13:37:43 +0000 Subject: [PATCH 094/145] Convert Tarball easyblock to run_shell_cmd --- easybuild/easyblocks/generic/tarball.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/generic/tarball.py b/easybuild/easyblocks/generic/tarball.py index 5ad887cc81..0d656cc2e8 100644 --- a/easybuild/easyblocks/generic/tarball.py +++ b/easybuild/easyblocks/generic/tarball.py @@ -41,7 +41,7 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import copy_dir, extract_file, remove_dir -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class Tarball(ExtensionEasyBlock): @@ -94,7 +94,7 @@ def install_step(self, src=None): preinstall_cmd = '&& '.join([cmd for cmd in [preinstall_cmd, self.cfg['preinstall_cmd']] if cmd]) if preinstall_cmd: self.log.info("Preparing installation of %s using command '%s'..." % (self.name, preinstall_cmd)) - run_cmd(preinstall_cmd, log_all=True, simple=True) + run_shell_cmd(preinstall_cmd) # Copy source directory source_path = src or self.cfg['start_dir'] From d7c6474156977e5217900f5cb65f9da7f34187ab Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Tue, 12 Dec 2023 14:35:42 +0000 Subject: [PATCH 095/145] Convert PerlModule easyblock to run_shell_cmd --- easybuild/easyblocks/generic/perlmodule.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/easybuild/easyblocks/generic/perlmodule.py b/easybuild/easyblocks/generic/perlmodule.py index e10b6d5b1c..ff3c1bbf50 100644 --- a/easybuild/easyblocks/generic/perlmodule.py +++ b/easybuild/easyblocks/generic/perlmodule.py @@ -35,7 +35,7 @@ from easybuild.framework.extensioneasyblock import ExtensionEasyBlock from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.environment import unset_env_vars @@ -79,7 +79,7 @@ def install_perl_module(self): '%s=%s' % (prefix_opt, self.installdir), self.cfg['configopts'], ]) - run_cmd(install_cmd) + run_shell_cmd(install_cmd) ConfigureMake.build_step(self) ConfigureMake.test_step(self) @@ -98,13 +98,13 @@ def install_perl_module(self): self.installdir, self.cfg['configopts'], ]) - run_cmd(install_cmd) + run_shell_cmd(install_cmd) - run_cmd("%s perl Build build %s" % (self.cfg['prebuildopts'], self.cfg['buildopts'])) + run_shell_cmd("%s perl Build build %s" % (self.cfg['prebuildopts'], self.cfg['buildopts'])) if self.cfg['runtest']: - run_cmd('perl Build %s' % self.cfg['runtest']) - run_cmd('%s perl Build install %s' % (self.cfg['preinstallopts'], self.cfg['installopts'])) + run_shell_cmd('perl Build %s' % self.cfg['runtest']) + run_shell_cmd('%s perl Build install %s' % (self.cfg['preinstallopts'], self.cfg['installopts'])) def run(self): """Perform the actual Perl module build/installation procedure""" From 52fa3c33f6dcb25241370cf0693aa173f42a1805 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Tue, 12 Dec 2023 14:40:47 +0000 Subject: [PATCH 096/145] Convert RPackage easyblock to run_shell_cmd --- easybuild/easyblocks/generic/rpackage.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/easybuild/easyblocks/generic/rpackage.py b/easybuild/easyblocks/generic/rpackage.py index 897408503c..89c0901cda 100644 --- a/easybuild/easyblocks/generic/rpackage.py +++ b/easybuild/easyblocks/generic/rpackage.py @@ -41,7 +41,7 @@ from easybuild.framework.extensioneasyblock import ExtensionEasyBlock from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import mkdir, copy_file -from easybuild.tools.run import run_cmd, parse_log_for_error +from easybuild.tools.run import run_shell_cmd, parse_log_for_error def make_R_install_option(opt, values, cmdline=False): @@ -168,8 +168,8 @@ def build_step(self): def install_R_package(self, cmd, inp=None): """Install R package as specified, and check for errors.""" - output, _ = run_cmd(cmd, log_all=True, simple=False, inp=inp, regexp=False) - self.check_install_output(output) + res = run_shell_cmd(cmd, stdin=inp) + self.check_install_output(res.output) def check_install_output(self, output): """ @@ -184,7 +184,7 @@ def check_install_output(self, output): """ % self.name # remove package if errors were detected # it's possible that some of the dependencies failed, but the package itself was installed - run_cmd(cmd, log_all=False, log_ok=False, simple=False, inp=stdin, regexp=False) + run_shell_cmd(cmd, stdin=stdin) raise EasyBuildError("Errors detected during installation of R package %s!", self.name) else: self.log.debug("R package %s installed succesfully", self.name) @@ -216,11 +216,11 @@ def required_deps(self): if self._required_deps is None: if self.src: cmd = "tar --wildcards --extract --file %s --to-stdout '*/DESCRIPTION'" % self.src - out, _ = run_cmd(cmd, simple=False, trace=False) + res = run_shell_cmd(cmd, hidden=True) # lines that start with whitespace are merged with line above lines = [] - for line in out.splitlines(): + for line in res.output.splitlines(): if line and line[0] in (' ', '\t'): lines[-1] = lines[-1] + line else: @@ -271,8 +271,8 @@ def prepare_r_ext_install(self): # determine location if isinstance(self.master, EB_R): # extension is being installed as part of an R installation/module - (out, _) = run_cmd("R RHOME", log_all=True, simple=False, trace=False) - rhome = out.strip() + res = run_shell_cmd("R RHOME", hidden=True) + rhome = res.output.strip() lib_install_prefix = os.path.join(rhome, 'library') else: # extension is being installed in a separate installation prefix From 30a907e46ff937741c8b477c1a467e55fcaed3f5 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Tue, 12 Dec 2023 14:42:16 +0000 Subject: [PATCH 097/145] Convert Cargo easyblock to run_shell_cmd --- easybuild/easyblocks/generic/cargo.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/easybuild/easyblocks/generic/cargo.py b/easybuild/easyblocks/generic/cargo.py index c47f2e9f05..3559da7eaf 100644 --- a/easybuild/easyblocks/generic/cargo.py +++ b/easybuild/easyblocks/generic/cargo.py @@ -36,7 +36,7 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.extensioneasyblock import ExtensionEasyBlock from easybuild.tools.filetools import extract_file, change_dir -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.config import build_option from easybuild.tools.filetools import write_file, compute_checksum from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC @@ -199,7 +199,7 @@ def build_step(self): if self.cfg['lto'] is not None: lto = '--config profile.%s.lto=\\"%s\\"' % (self.profile, self.cfg['lto']) - run_cmd('rustc --print cfg', log_all=True, simple=True) # for tracking in log file + run_shell_cmd('rustc --print cfg') # for tracking in log file cmd = ' '.join([ self.cfg['prebuildopts'], 'cargo build', @@ -209,7 +209,7 @@ def build_step(self): parallel, self.cfg['buildopts'], ]) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) def test_step(self): """Test with cargo""" @@ -220,7 +220,7 @@ def test_step(self): '--profile=' + self.profile, self.cfg['testopts'], ]) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) def install_step(self): """Install with cargo""" @@ -232,7 +232,7 @@ def install_step(self): '--path=.', self.cfg['installopts'], ]) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) def generate_crate_list(sourcedir): From c039ab41c85c66cd52a3ed87c9a9845dab5b287a Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Tue, 12 Dec 2023 14:44:09 +0000 Subject: [PATCH 098/145] Convert Conda easyblock to run_shell_cmd --- easybuild/easyblocks/generic/conda.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/easybuild/easyblocks/generic/conda.py b/easybuild/easyblocks/generic/conda.py index 03e78ab42e..3ad69f1f6e 100644 --- a/easybuild/easyblocks/generic/conda.py +++ b/easybuild/easyblocks/generic/conda.py @@ -33,7 +33,7 @@ from easybuild.easyblocks.generic.binary import Binary from easybuild.framework.easyconfig import CUSTOM -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.modules import get_software_root from easybuild.tools.build_log import EasyBuildError @@ -74,7 +74,7 @@ def install_step(self): # initialize conda environment # setuptools is just a choice, but *something* needs to be there cmd = "%s config --add create_default_packages setuptools" % conda_cmd - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) if self.cfg['environment_file'] or self.cfg['remote_environment']: @@ -86,7 +86,7 @@ def install_step(self): # use --force to ignore existing installation directory cmd = "%s %s env create --force %s -p %s" % (self.cfg['preinstallopts'], conda_cmd, env_spec, self.installdir) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) else: @@ -100,11 +100,11 @@ def install_step(self): cmd = "%s %s create --force -y -p %s %s" % (self.cfg['preinstallopts'], conda_cmd, self.installdir, install_args) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) # clean up cmd = "%s clean -ya" % conda_cmd - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) def make_module_extra(self): """Add the install directory to the PATH.""" From d89524cacd8aab833d5620c0710dc9dfdbecab0b Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Tue, 12 Dec 2023 14:46:34 +0000 Subject: [PATCH 099/145] Convert GoPackage easyblock to run_shell_cmd --- easybuild/easyblocks/generic/gopackage.py | 26 +++++++++++------------ 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/easybuild/easyblocks/generic/gopackage.py b/easybuild/easyblocks/generic/gopackage.py index 8a3b591402..b829159349 100644 --- a/easybuild/easyblocks/generic/gopackage.py +++ b/easybuild/easyblocks/generic/gopackage.py @@ -35,7 +35,7 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError from easybuild.tools.modules import get_software_root, get_software_version -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class GoPackage(EasyBlock): @@ -71,8 +71,8 @@ def configure_step(self): env.setvar('GOBIN', os.path.join(self.installdir, 'bin'), verbose=False) # creates log entries for go being used, for debugging - run_cmd("go version", verbose=False, trace=False) - run_cmd("go env", verbose=False, trace=False) + run_shell_cmd("go version", hidden=True) + run_shell_cmd("go env", hidden=True) def build_step(self): """If Go package is not native go module, lets try to make the module.""" @@ -91,13 +91,13 @@ def build_step(self): # go mod init cmd = ' '.join(['go', 'mod', 'init', self.cfg['modulename']]) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) if self.cfg['forced_deps']: for dep in self.cfg['forced_deps']: # go get specific dependencies which locks them in go.mod cmd = ' '.join(['go', 'get', '%s@%s' % dep]) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) # note: ... (tripledot) used below is not a typo, but go wildcard pattern # which means: anything you can find in this directory, including all subdirectories @@ -105,20 +105,20 @@ def build_step(self): # see: https://stackoverflow.com/a/28031651/2047157 # building and testing will add packages to go.mod - run_cmd('go build ./...', log_all=True, simple=True) - run_cmd('go test ./...', log_all=True, simple=True) + run_shell_cmd('go build ./...') + run_shell_cmd('go test ./...') # tidy up go.mod - run_cmd('go mod tidy', log_all=True, simple=True) + run_shell_cmd('go mod tidy') # build and test again, to ensure go mod tidy didn't removed anything needed - run_cmd('go build ./...', log_all=True, simple=True) - run_cmd('go test ./...', log_all=True, simple=True) + run_shell_cmd('go build ./...') + run_shell_cmd('go test ./...') self.log.warn('Include generated go.mod and go.sum via patch to ensure locked dependencies ' 'and run this easyconfig again.') - run_cmd('cat go.mod', log_all=True, simple=True) - run_cmd('cat go.sum', log_all=True, simple=True) + run_shell_cmd('cat go.mod') + run_shell_cmd('cat go.sum') if not os.path.exists(go_sum_file) or not os.path.isfile(go_sum_file): raise EasyBuildError("go.sum not found! This module has no locked dependency versions.") @@ -136,7 +136,7 @@ def install_step(self): '-x', self.cfg['installopts'], ]) - run_cmd(cmd, log_all=True, log_ok=True, simple=True) + run_shell_cmd(cmd) def sanity_check_step(self): """Custom sanity check for Go package.""" From 816af4d65b10e50211073a581bd0953fab4c8fe3 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Tue, 12 Dec 2023 14:48:15 +0000 Subject: [PATCH 100/145] Convert JuliaPackage easyblock to run_shell_cmd --- easybuild/easyblocks/generic/juliapackage.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/generic/juliapackage.py b/easybuild/easyblocks/generic/juliapackage.py index 2ff1e49e89..90a2c6f139 100644 --- a/easybuild/easyblocks/generic/juliapackage.py +++ b/easybuild/easyblocks/generic/juliapackage.py @@ -38,7 +38,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.modules import get_software_root, get_software_version from easybuild.tools.filetools import copy_dir -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd EXTS_FILTER_JULIA_PACKAGES = ("julia -e 'using %(ext_name)s'", "") USER_DEPOT_PATTERN = re.compile(r"\/\.julia\/?$") @@ -147,9 +147,9 @@ def install_step(self): "julia -e '%s'" % julia_pkg_cmd, self.cfg['installopts'], ]) - (out, _) = run_cmd(cmd, log_all=True, simple=False) + res = run_shell_cmd(cmd) - return out + return res.output def run(self): """Install Julia package as an extension.""" From 853a9f7c1da74fc582389afd255369b3e323724f Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Tue, 12 Dec 2023 14:49:12 +0000 Subject: [PATCH 101/145] Convert OCamlPackage easyblock to run_shell_cmd --- easybuild/easyblocks/generic/ocamlpackage.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/generic/ocamlpackage.py b/easybuild/easyblocks/generic/ocamlpackage.py index d0b460d2ca..4010006c7e 100644 --- a/easybuild/easyblocks/generic/ocamlpackage.py +++ b/easybuild/easyblocks/generic/ocamlpackage.py @@ -29,7 +29,7 @@ """ from easybuild.framework.extensioneasyblock import ExtensionEasyBlock from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class OCamlPackage(ExtensionEasyBlock): @@ -42,8 +42,8 @@ def configure_step(self): def run(self): """Perform OCaml package installation (as extension).""" # install using 'opam install' - run_cmd("eval `opam config env` && opam install -yv %s.%s" % (self.name, self.version)) + run_shell_cmd("eval `opam config env` && opam install -yv %s.%s" % (self.name, self.version)) # 'opam pin add' fixes the version of the package # see https://opam.ocaml.org/doc/Usage.html#opampin - run_cmd("eval `opam config env` && opam pin -yv add %s %s" % (self.name, self.version)) + run_shell_cmd("eval `opam config env` && opam pin -yv add %s %s" % (self.name, self.version)) From 8b4d0c3d140de62bc2be40afa43ae3f1f172cc0a Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Tue, 12 Dec 2023 14:50:09 +0000 Subject: [PATCH 102/145] Convert OctavePackage easyblock to run_shell_cmd --- easybuild/easyblocks/generic/octavepackage.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/generic/octavepackage.py b/easybuild/easyblocks/generic/octavepackage.py index cff63cc9c6..cbe26f3287 100644 --- a/easybuild/easyblocks/generic/octavepackage.py +++ b/easybuild/easyblocks/generic/octavepackage.py @@ -34,7 +34,7 @@ from easybuild.framework.extensioneasyblock import ExtensionEasyBlock from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import change_dir -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class OctavePackage(ExtensionEasyBlock): @@ -56,7 +56,7 @@ def run(self): # create temporary tarball from unpacked & patched source src = os.path.join(tempfile.gettempdir(), '%s-%s-patched.tar.gz' % (self.name, self.version)) cwd = change_dir(os.path.dirname(self.ext_dir)) - run_cmd("tar cfvz %s %s" % (src, os.path.basename(self.ext_dir))) + run_shell_cmd("tar cfvz %s %s" % (src, os.path.basename(self.ext_dir))) change_dir(cwd) else: src = self.src @@ -69,4 +69,4 @@ def run(self): octave_cmd += "pkg install -global %s" % src - run_cmd("octave --eval '%s'" % octave_cmd) + run_shell_cmd("octave --eval '%s'" % octave_cmd) From a87daf225a38e7bf0cb299731ea1837988730c09 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Tue, 12 Dec 2023 14:51:11 +0000 Subject: [PATCH 103/145] Convert RubyGem easyblock to run_shell_cmd --- easybuild/easyblocks/generic/rubygem.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/generic/rubygem.py b/easybuild/easyblocks/generic/rubygem.py index 58ecfa05c4..a989b91bf7 100644 --- a/easybuild/easyblocks/generic/rubygem.py +++ b/easybuild/easyblocks/generic/rubygem.py @@ -36,7 +36,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import copy_file from easybuild.tools.modules import get_software_root -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class RubyGem(ExtensionEasyBlock): @@ -113,7 +113,7 @@ def install_step(self): env.setvar('GEM_HOME', self.installdir) bindir = os.path.join(self.installdir, 'bin') - run_cmd("gem install --bindir %s --local %s" % (bindir, self.ext_src)) + run_shell_cmd("gem install --bindir %s --local %s" % (bindir, self.ext_src)) def make_module_extra(self): """Extend $GEM_PATH in module file.""" From bce0644c545968d1c758e04500ae243e4111826d Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Tue, 12 Dec 2023 15:46:24 +0000 Subject: [PATCH 104/145] Convert IntelBase easyblock to run_shell_cmd --- easybuild/easyblocks/generic/intelbase.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/easybuild/easyblocks/generic/intelbase.py b/easybuild/easyblocks/generic/intelbase.py index 154b7c86ea..21e7a2989a 100644 --- a/easybuild/easyblocks/generic/intelbase.py +++ b/easybuild/easyblocks/generic/intelbase.py @@ -49,7 +49,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import adjust_permissions, find_flexlm_license from easybuild.tools.filetools import mkdir, read_file, remove_file, write_file -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd # different supported activation types (cfr. Intel documentation) @@ -408,7 +408,7 @@ def install_step_classic(self, silent_cfg_names_map=None, silent_cfg_extras=None self.cfg['installopts'], ]) - return run_cmd(cmd, log_all=True, simple=True, log_output=True) + run_shell_cmd(cmd) def install_step_oneapi(self, *args, **kwargs): """ @@ -448,16 +448,16 @@ def install_step_oneapi(self, *args, **kwargs): cmd.append(self.cfg['installopts']) - return run_cmd(' '.join(cmd), log_all=True, simple=True, log_output=True) + run_shell_cmd(' '.join(cmd)) def install_step(self, *args, **kwargs): """ Install Intel software """ if LooseVersion(self.version) >= LooseVersion('2021'): - return self.install_step_oneapi(*args, **kwargs) + self.install_step_oneapi(*args, **kwargs) else: - return self.install_step_classic(*args, **kwargs) + self.install_step_classic(*args, **kwargs) def move_after_install(self): """Move installed files to correct location after installation.""" From 73364a6255053baa25e465e2b208793e49ea09ce Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Tue, 12 Dec 2023 15:47:09 +0000 Subject: [PATCH 105/145] Convert Rpm easyblock to run_shell_cmd --- easybuild/easyblocks/generic/rpm.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/easybuild/easyblocks/generic/rpm.py b/easybuild/easyblocks/generic/rpm.py index 1ab63bfff1..5156f716ea 100644 --- a/easybuild/easyblocks/generic/rpm.py +++ b/easybuild/easyblocks/generic/rpm.py @@ -46,7 +46,7 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import change_dir, mkdir, symlink, which -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd _log = fancylogger.getLogger('easyblocks.generic.rpm') @@ -88,7 +88,7 @@ def rebuild_rpm(rpm_path, targetdir): targetdir, rpm_path, ]) - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) class Rpm(Binary): @@ -142,10 +142,10 @@ def configure_step(self): # determine whether RPMs need to be rebuilt to make relocation work cmd = "rpm --version" - (out, _) = run_cmd(cmd, log_all=True, simple=False) + res = run_shell_cmd(cmd) rpmver_re = re.compile(r"^RPM\s+version\s+(?P[0-9.]+).*") - res = rpmver_re.match(out) + res = rpmver_re.match(res.output) self.log.debug("RPM version found: %s" % res.group()) if res: @@ -185,7 +185,7 @@ def install_step(self): cmd = "rpm --initdb --dbpath /rpm --root %s" % self.installdir - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) force = '' if self.cfg['force']: @@ -219,7 +219,7 @@ def install_step(self): 'post': postinstall, 'installopts': self.cfg['installopts'], } - run_cmd(cmd, log_all=True, simple=True) + run_shell_cmd(cmd) for path in self.cfg['makesymlinks']: # allow globs, always use first hit. From ff8e9ecb4b2b4ea64f17fea713b84b10a54504f6 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Tue, 12 Dec 2023 15:47:26 +0000 Subject: [PATCH 106/145] Convert SCons easyblock to run_shell_cmd --- easybuild/easyblocks/generic/scons.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/easybuild/easyblocks/generic/scons.py b/easybuild/easyblocks/generic/scons.py index 9820cb5855..b581bf0152 100644 --- a/easybuild/easyblocks/generic/scons.py +++ b/easybuild/easyblocks/generic/scons.py @@ -29,7 +29,7 @@ """ from easybuild.framework.easyblock import EasyBlock from easybuild.framework.easyconfig import CUSTOM -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class SCons(EasyBlock): @@ -66,9 +66,9 @@ def build_step(self, verbose=False): 'prefix': self.prefix, 'par': par, } - (out, _) = run_cmd(cmd, log_all=True, log_output=verbose) + res = run_shell_cmd(cmd) - return out + return res.output def test_step(self): """ @@ -76,7 +76,7 @@ def test_step(self): """ if self.cfg['runtest']: cmd = "%s scons %s %s" % (self.cfg['pretestopts'], self.cfg['runtest'], self.cfg['testopts']) - run_cmd(cmd, log_all=True) + run_shell_cmd(cmd) def install_step(self): """ @@ -87,6 +87,6 @@ def install_step(self): 'preinstallopts': self.cfg['preinstallopts'], 'prefix': self.prefix, } - (out, _) = run_cmd(cmd, log_all=True) + res = run_shell_cmd(cmd) - return out + return res.output From e43be23fb9da90fc2555c4546a7ef639824892ab Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Tue, 12 Dec 2023 15:48:08 +0000 Subject: [PATCH 107/145] Convert SystemCompiler easyblock to run_shell_cmd --- easybuild/easyblocks/generic/systemcompiler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/generic/systemcompiler.py b/easybuild/easyblocks/generic/systemcompiler.py index 0b01c30036..d75e357dbe 100644 --- a/easybuild/easyblocks/generic/systemcompiler.py +++ b/easybuild/easyblocks/generic/systemcompiler.py @@ -41,7 +41,7 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.filetools import read_file, resolve_path, which -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd _log = fancylogger.getLogger('easyblocks.generic.systemcompiler') @@ -54,8 +54,8 @@ def extract_compiler_version(compiler_name): # Intel(R) C Intel(R) 64 Compiler XE for applications running on Intel(R) 64, Version 15.0.1.133 Build 20141023 version_regex = re.compile(r'\s([0-9]+(?:\.[0-9]+){1,3})\s', re.M) if compiler_name == 'gcc': - out, _ = run_cmd("gcc --version", simple=False) - res = version_regex.search(out) + res = run_shell_cmd("gcc --version") + res = version_regex.search(res.output) if res is None: raise EasyBuildError("Could not extract GCC version from %s", out) compiler_version = res.group(1) From 1a7fe1085115fb8ec375730b3e359c6034165acc Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Tue, 12 Dec 2023 15:48:39 +0000 Subject: [PATCH 108/145] Convert SystemMPI easyblock to run_shell_cmd --- easybuild/easyblocks/generic/systemmpi.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/easybuild/easyblocks/generic/systemmpi.py b/easybuild/easyblocks/generic/systemmpi.py index 5ba5a3f812..5a71d058a1 100644 --- a/easybuild/easyblocks/generic/systemmpi.py +++ b/easybuild/easyblocks/generic/systemmpi.py @@ -38,7 +38,7 @@ from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.filetools import read_file, resolve_path, which from easybuild.tools.modules import get_software_version -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class SystemMPI(Bundle, ConfigureMake, EB_impi): @@ -124,7 +124,8 @@ def prepare_step(self, *args, **kwargs): # Determine MPI version, installation prefix and underlying compiler if mpi_name in ('openmpi', 'spectrummpi'): # Spectrum MPI is based on Open MPI so is also covered by this logic - output_of_ompi_info, _ = run_cmd("ompi_info", simple=False) + res = run_shell_cmd("ompi_info") + output_of_ompi_info = res.output # Extract the version of the MPI implementation if mpi_name == 'spectrummpi': @@ -183,8 +184,9 @@ def prepare_step(self, *args, **kwargs): self.mpi_env_vars[key] = value # Extract the C compiler used underneath Intel MPI - compile_info, exit_code = run_cmd("%s -compile-info" % mpi_c_wrapper, simple=False) - if exit_code == 0: + res == run_shell_cmd("%s -compile-info" % mpi_c_wrapper) + compile_info = res.output + if res.exit_code == 0: self.mpi_c_compiler = compile_info.split(' ', 1)[0] else: raise EasyBuildError("Could not determine C compiler underneath Intel MPI, '%s -compiler-info' " From 40d10fa4e4b8074dcbc81f07f8a35e0b9ce287e9 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Tue, 12 Dec 2023 15:48:58 +0000 Subject: [PATCH 109/145] Convert Waf easyblock to run_shell_cmd --- easybuild/easyblocks/generic/waf.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/easybuild/easyblocks/generic/waf.py b/easybuild/easyblocks/generic/waf.py index 30d36a72dd..6ce9541adc 100644 --- a/easybuild/easyblocks/generic/waf.py +++ b/easybuild/easyblocks/generic/waf.py @@ -29,7 +29,7 @@ """ from easybuild.framework.easyblock import EasyBlock -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class Waf(EasyBlock): @@ -48,9 +48,9 @@ def configure_step(self, cmd_prefix=''): '--prefix=%s' % self.installdir, self.cfg['configopts'], ]) - (out, _) = run_cmd(cmd, log_all=True, simple=False) + res = run_shell_cmd(cmd) - return out + return res.output def build_step(self, verbose=False, path=None): """ @@ -62,9 +62,9 @@ def build_step(self, verbose=False, path=None): 'build', self.cfg['buildopts'], ]) - (out, _) = run_cmd(cmd, log_all=True, simple=False) + res = run_shell_cmd(cmd) - return out + return res.output def install_step(self, verbose=False, path=None): """ @@ -76,6 +76,6 @@ def install_step(self, verbose=False, path=None): 'install', self.cfg['installopts'], ]) - (out, _) = run_cmd(cmd, log_all=True, simple=False) + res = run_shell_cmd(cmd) - return out + return res.output From f402ac94a7fec9d06881d7e90d7d4184815e1c0a Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Tue, 12 Dec 2023 16:00:21 +0000 Subject: [PATCH 110/145] pythonpackage: log_ok=False -> fail_on_error=False to use ec --- easybuild/easyblocks/generic/pythonpackage.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index e65c712c3b..520b171230 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -831,10 +831,13 @@ def test_step(self, return_output_ec=False): self.cfg['testopts'], ]) - res = run_shell_cmd(cmd) if return_output_ec: + res = run_shell_cmd(cmd, fail_on_error=False) + # need to retrieve ec by not failing on error (out, ec) = (res.output, res.exit_code) self.log.info("cmd '%s' exited with exit code %s and output:\n%s", cmd, ec, out) + else: + run_shell_cmd(cmd) if test_installdir: remove_dir(test_installdir) @@ -1022,7 +1025,7 @@ def sanity_check_step(self, *args, **kwargs): pip_check_errors = [] - res = run_shell_cmd(pip_check_command) + res = run_shell_cmd(pip_check_command, fail_on_error=False) pip_check_msg = res.output if res.exit_code: pip_check_errors.append('`%s` failed:\n%s' % (pip_check_command, pip_check_msg)) From 5e71a44bac509a1d1131a9536a4d4e7aa7c8723e Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Tue, 12 Dec 2023 16:03:20 +0000 Subject: [PATCH 111/145] rpackage: log_ok=False -> fail_on_error=False --- easybuild/easyblocks/generic/rpackage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/rpackage.py b/easybuild/easyblocks/generic/rpackage.py index 89c0901cda..bc43ce6970 100644 --- a/easybuild/easyblocks/generic/rpackage.py +++ b/easybuild/easyblocks/generic/rpackage.py @@ -184,7 +184,7 @@ def check_install_output(self, output): """ % self.name # remove package if errors were detected # it's possible that some of the dependencies failed, but the package itself was installed - run_shell_cmd(cmd, stdin=stdin) + run_shell_cmd(cmd, fail_on_error=False, stdin=stdin) raise EasyBuildError("Errors detected during installation of R package %s!", self.name) else: self.log.debug("R package %s installed succesfully", self.name) From 6f6981355a58abbc960bce958fd3181b7ef0811d Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Tue, 12 Dec 2023 16:07:05 +0000 Subject: [PATCH 112/145] Fix use of `out` in systemcompiler.py --- easybuild/easyblocks/generic/systemcompiler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/systemcompiler.py b/easybuild/easyblocks/generic/systemcompiler.py index d75e357dbe..46d800731f 100644 --- a/easybuild/easyblocks/generic/systemcompiler.py +++ b/easybuild/easyblocks/generic/systemcompiler.py @@ -55,7 +55,8 @@ def extract_compiler_version(compiler_name): version_regex = re.compile(r'\s([0-9]+(?:\.[0-9]+){1,3})\s', re.M) if compiler_name == 'gcc': res = run_shell_cmd("gcc --version") - res = version_regex.search(res.output) + out = res.output + res = version_regex.search(out) if res is None: raise EasyBuildError("Could not extract GCC version from %s", out) compiler_version = res.group(1) From 35871bd6370d1237f2475602cfcc5df98f6dbf3d Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 13 Dec 2023 18:37:43 +0100 Subject: [PATCH 113/145] update Ferret easyblock to be compatible with v7.6.0 --- easybuild/easyblocks/f/ferret.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/f/ferret.py b/easybuild/easyblocks/f/ferret.py index 8bde8f5641..1b692b485f 100644 --- a/easybuild/easyblocks/f/ferret.py +++ b/easybuild/easyblocks/f/ferret.py @@ -172,6 +172,14 @@ def configure_step(self): regex_subs.append((r"^(\s*%s\s*)=.*" % key, r"\1 = %s" % os.getenv(value))) if LooseVersion(self.version) >= LooseVersion("7.3"): + flag_vars = { + "CFLAGS": "CFLAGS", + "FFLAGS": "FFLAGS", + "PPLUS_FFLAGS": "FFLAGS", + } + for key, value in flag_vars.items(): + regex_subs.append((r"^(\s*%s\s*=).*-m64 (.*)" % key, r"\1%s \2" % os.getenv(value))) + regex_subs.extend([ (r"^(\s*LDFLAGS\s*=).*", r"\1 -fPIC %s -lnetcdff -lnetcdf -lhdf5_hl -lhdf5" % os.getenv("LDFLAGS")), (r"^(\s*)CDFLIB", r"\1NONEED"), @@ -182,8 +190,6 @@ def configure_step(self): for x in ["CFLAGS", "FFLAGS"]: regex_subs.append((r"^(\s*%s\s*=\s*\$\(CPP_FLAGS\)).*\\" % x, r"\1 %s \\" % os.getenv(x))) if LooseVersion(self.version) >= LooseVersion("7.3"): - for x in ["CFLAGS", "FFLAGS"]: - regex_subs.append((r"^(\s*%s\s*=).*-m64 (.*)" % x, r"\1%s \2" % os.getenv(x))) regex_subs.extend(sorted(gfort2ifort.items())) regex_subs.append((r"^(\s*MYDEFINES\s*=.*)\\", r"\1-DF90_SYSTEM_ERROR_CALLS \\")) @@ -208,6 +214,9 @@ def sanity_check_step(self): """Custom sanity check for Ferret.""" major_minor_version = '.'.join(self.version.split('.')[:2]) + if LooseVersion(self.version) >= LooseVersion("7.6"): + major_minor_version += self.version.split('.')[2] + custom_paths = { 'files': ["bin/ferret_v%s" % major_minor_version], 'dirs': [], From bb9ffc046c6781635fe44124125373f6bb6f910b Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 14 Dec 2023 14:19:31 +0100 Subject: [PATCH 114/145] Add binutils symlinks when building TensorFlow with --rpath Avoids build errors like > # Configuration: 59a0a0e80bc61398ca9fef36ddd3d195f45bc65f8bfbc1a894cf677de7d5d5e9 > # Execution platform: @local_execution_config_platform//:platform > src/main/tools/process-wrapper-legacy.cc:80: "execvp(/tmp/eb-m2n16g8t/tmptk5axscj/rpath_wrappers/ld_wrapper/ar, ...)": No such file or directory > Target //tensorflow/tools/pip_package:build_pip_package failed to build --- easybuild/easyblocks/t/tensorflow.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/t/tensorflow.py b/easybuild/easyblocks/t/tensorflow.py index 926a1920cf..23edf3cf10 100644 --- a/easybuild/easyblocks/t/tensorflow.py +++ b/easybuild/easyblocks/t/tensorflow.py @@ -45,7 +45,7 @@ from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.config import build_option, IGNORE from easybuild.tools.filetools import adjust_permissions, apply_regex_substitutions, copy_file, mkdir, resolve_path -from easybuild.tools.filetools import is_readable, read_file, which, write_file, remove_file +from easybuild.tools.filetools import is_readable, read_file, symlink, which, write_file, remove_file from easybuild.tools.modules import get_software_root, get_software_version, get_software_libdir from easybuild.tools.run import run_cmd from easybuild.tools.systemtools import AARCH64, X86_64, get_cpu_architecture, get_os_name, get_os_version @@ -449,6 +449,8 @@ def setup_build_dirs(self): def configure_step(self): """Custom configuration procedure for TensorFlow.""" + self.setup_build_dirs() + # Bazel seems to not be able to handle a large amount of parallel jobs, e.g. 176 on some Power machines, # and will hang forever building the TensorFlow package. # So limit to something high but still reasonable while allowing ECs to overwrite it @@ -461,6 +463,19 @@ def configure_step(self): # note that this may be an RPATH wrapper script (when EasyBuild is configured with --rpath) ld_path = which('ld') self.binutils_bin_path = os.path.dirname(ld_path) + if self.toolchain.is_rpath_wrapper(ld_path): + if os.path.basename(os.path.dirname(os.path.dirname(ld_path))) == toolchain.RPATH_WRAPPERS_SUBDIR: + # TF expects all binutils binaries in a single path but newer EB puts each in their own subfolder + # Add symlinks to each binutils binary into a single folder + new_rpath_wrapper_dir = os.path.join(self.wrapper_dir, toolchain.RPATH_WRAPPERS_SUBDIR) + binutils_bin_path = os.path.join(get_software_root('binutils'), 'bin') + self.log.info("Found %s to be an rpath wrapper. Adding symlinks for binutils in %s to %s.", + ld_path, binutils_bin_path, new_rpath_wrapper_dir) + mkdir(new_rpath_wrapper_dir) + for file in next(os.walk(binutils_bin_path))[2]: + # Note the use of `which` to take rpath wrappers where available + symlink(which(file), os.path.join(new_rpath_wrapper_dir, file)) + self.binutils_bin_path = new_rpath_wrapper_dir # filter out paths from CPATH and LIBRARY_PATH. This is needed since bazel will pull some dependencies that # might conflict with dependencies on the system and/or installed with EB. For example: protobuf @@ -473,8 +488,6 @@ def configure_step(self): filtered_path = os.pathsep.join([p for fil in path_filter for p in path if fil not in p]) env.setvar(var, filtered_path) - self.setup_build_dirs() - use_wrapper = False if self.toolchain.comp_family() == toolchain.INTELCOMP: # put wrappers for Intel C/C++ compilers in place (required to make sure license server is found) From 4ad54893f5dd1f9f880dea6c6343d4cd37a44855 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 14 Dec 2023 14:52:29 +0100 Subject: [PATCH 115/145] Import RPATH_WRAPPERS_SUBDIR --- easybuild/easyblocks/t/tensorflow.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/t/tensorflow.py b/easybuild/easyblocks/t/tensorflow.py index 23edf3cf10..273c26bd06 100644 --- a/easybuild/easyblocks/t/tensorflow.py +++ b/easybuild/easyblocks/t/tensorflow.py @@ -49,6 +49,7 @@ from easybuild.tools.modules import get_software_root, get_software_version, get_software_libdir from easybuild.tools.run import run_cmd from easybuild.tools.systemtools import AARCH64, X86_64, get_cpu_architecture, get_os_name, get_os_version +from easybuild.tools.toolchain.toolchain import RPATH_WRAPPERS_SUBDIR CPU_DEVICE = 'cpu' @@ -464,10 +465,10 @@ def configure_step(self): ld_path = which('ld') self.binutils_bin_path = os.path.dirname(ld_path) if self.toolchain.is_rpath_wrapper(ld_path): - if os.path.basename(os.path.dirname(os.path.dirname(ld_path))) == toolchain.RPATH_WRAPPERS_SUBDIR: + if os.path.basename(os.path.dirname(os.path.dirname(ld_path))) == RPATH_WRAPPERS_SUBDIR: # TF expects all binutils binaries in a single path but newer EB puts each in their own subfolder # Add symlinks to each binutils binary into a single folder - new_rpath_wrapper_dir = os.path.join(self.wrapper_dir, toolchain.RPATH_WRAPPERS_SUBDIR) + new_rpath_wrapper_dir = os.path.join(self.wrapper_dir, RPATH_WRAPPERS_SUBDIR) binutils_bin_path = os.path.join(get_software_root('binutils'), 'bin') self.log.info("Found %s to be an rpath wrapper. Adding symlinks for binutils in %s to %s.", ld_path, binutils_bin_path, new_rpath_wrapper_dir) From 2ac33cfca44e6e5f0ed21bca78ac13a6a892571e Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 14 Dec 2023 16:11:18 +0100 Subject: [PATCH 116/145] Create wrapper_dir Required for the RPath wrappers and also assumed to exist by `self.write_wrapper` --- easybuild/easyblocks/t/tensorflow.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/easyblocks/t/tensorflow.py b/easybuild/easyblocks/t/tensorflow.py index 273c26bd06..658dcca5e7 100644 --- a/easybuild/easyblocks/t/tensorflow.py +++ b/easybuild/easyblocks/t/tensorflow.py @@ -446,6 +446,7 @@ def setup_build_dirs(self): self.output_user_root_dir = os.path.join(parent_dir, 'bazel-root') # Folder where wrapper binaries can be placed, where required. TODO: Replace by --action_env cmds self.wrapper_dir = os.path.join(parent_dir, 'wrapper_bin') + mkdir(self.wrapper_dir) def configure_step(self): """Custom configuration procedure for TensorFlow.""" From 9c5ff3435949a6c198906d16e8dd4f5368303c87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Mon, 18 Dec 2023 17:07:15 +0000 Subject: [PATCH 117/145] Use attribute instead of kwargs to check for if sources should be checked for --- easybuild/easyblocks/generic/bundle.py | 2 +- easybuild/easyblocks/generic/cargopythonbundle.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/generic/bundle.py b/easybuild/easyblocks/generic/bundle.py index f52b575254..97e982405a 100644 --- a/easybuild/easyblocks/generic/bundle.py +++ b/easybuild/easyblocks/generic/bundle.py @@ -77,7 +77,7 @@ def __init__(self, *args, **kwargs): # list of EasyConfig instances of components for which to run sanity checks self.comp_cfgs_sanity_check = [] - check_for_sources = kwargs.get('check_for_sources', True) + check_for_sources = getattr(self, 'check_for_sources', True) # list of sources for bundle itself *must* be empty (unless overridden by subclass) if check_for_sources: if self.cfg['sources']: diff --git a/easybuild/easyblocks/generic/cargopythonbundle.py b/easybuild/easyblocks/generic/cargopythonbundle.py index a55602770f..50f0721671 100644 --- a/easybuild/easyblocks/generic/cargopythonbundle.py +++ b/easybuild/easyblocks/generic/cargopythonbundle.py @@ -50,8 +50,8 @@ def extra_options(extra_vars=None): def __init__(self, *args, **kwargs): """Constructor for CargoPythonBundle easyblock.""" - PythonBundle.__init__(self, *args, check_for_sources=False, **kwargs) - Cargo.__init__(self, *args, **kwargs) + self.check_for_sources = False + super(CargoPythonBundle, self).__init__(*args, **kwargs) def extract_step(self): """Specifically use the overloaded variant from Cargo as is populates vendored sources with checksums.""" From 0db52c7387dd35cf3695f629b5c2e78d46f2e619 Mon Sep 17 00:00:00 2001 From: Chia-Jung Hsu Date: Mon, 18 Dec 2023 17:21:53 +0000 Subject: [PATCH 118/145] Fix suitesparse path --- easybuild/easyblocks/n/numpy.py | 26 ++++++++++---------------- easybuild/easyblocks/p/petsc.py | 23 +++++++++++++++-------- easybuild/easyblocks/t/trilinos.py | 20 +++++++++----------- 3 files changed, 34 insertions(+), 35 deletions(-) diff --git a/easybuild/easyblocks/n/numpy.py b/easybuild/easyblocks/n/numpy.py index 8f5a049e0d..889faf9d2d 100644 --- a/easybuild/easyblocks/n/numpy.py +++ b/easybuild/easyblocks/n/numpy.py @@ -183,23 +183,17 @@ def get_libs_for_mkl(varname): suitesparseroot = get_software_root('SuiteSparse') if suitesparseroot: - amddir = os.path.join(suitesparseroot, 'AMD') - umfpackdir = os.path.join(suitesparseroot, 'UMFPACK') - if not os.path.exists(amddir) or not os.path.exists(umfpackdir): - raise EasyBuildError("Expected SuiteSparse subdirectories are not both there: %s, %s", - amddir, umfpackdir) - else: - extrasiteconfig += '\n'.join([ - "[amd]", - "library_dirs = %s" % os.path.join(amddir, 'Lib'), - "include_dirs = %s" % os.path.join(amddir, 'Include'), - "amd_libs = amd", - "[umfpack]", - "library_dirs = %s" % os.path.join(umfpackdir, 'Lib'), - "include_dirs = %s" % os.path.join(umfpackdir, 'Include'), - "umfpack_libs = umfpack", - ]) + extrasiteconfig += '\n'.join([ + "[amd]", + "library_dirs = %s" % os.path.join(suitesparseroot, 'lib'), + "include_dirs = %s" % os.path.join(suitesparseroot, 'include'), + "amd_libs = amd", + "[umfpack]", + "library_dirs = %s" % os.path.join(suitesparseroot, 'lib'), + "include_dirs = %s" % os.path.join(suitesparseroot, 'include'), + "umfpack_libs = umfpack", + ]) self.sitecfg = '\n'.join([self.sitecfg, extrasiteconfig]) diff --git a/easybuild/easyblocks/p/petsc.py b/easybuild/easyblocks/p/petsc.py index a658dd0bbf..2b116fd447 100644 --- a/easybuild/easyblocks/p/petsc.py +++ b/easybuild/easyblocks/p/petsc.py @@ -29,7 +29,7 @@ """ import os import re -from easybuild.tools import LooseVersion +from distutils.version import LooseVersion import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain @@ -255,18 +255,25 @@ def configure_step(self): # More libraries added after version 3.17 if LooseVersion(self.version) >= LooseVersion("3.17"): # specified order of libs matters! + # ss_libs = ["UMFPACK", "KLU", "SPQR", "CHOLMOD", "BTF", "CCOLAMD", + # "COLAMD", "CSparse", "CXSparse", "LDL", "RBio", + # "SLIP_LU", "CAMD", "AMD"] + ss_libs = ["UMFPACK", "KLU", "SPQR", "CHOLMOD", "BTF", "CCOLAMD", - "COLAMD", "CSparse", "CXSparse", "LDL", "RBio", + "COLAMD", "CXSparse", "LDL", "RBio", "SLIP_LU", "CAMD", "AMD"] - suitesparse_inc = [os.path.join(suitesparse, x, "Include") - for x in ss_libs] - suitesparse_inc.append(os.path.join(suitesparse, "SuiteSparse_config")) + + # suitesparse_inc = [os.path.join(suitesparse, x, "Include") + # for x in ss_libs] + # suitesparse_inc.append(os.path.join(suitesparse, "SuiteSparse_config")) + suitesparse_inc = [os.path.join(suitesparse, "include")] inc_spec = "-include=[%s]" % ','.join(suitesparse_inc) - suitesparse_libs = [os.path.join(suitesparse, x, "Lib", "lib%s.a" % x.replace("_", "").lower()) - for x in ss_libs] - suitesparse_libs.append(os.path.join(suitesparse, "SuiteSparse_config", "libsuitesparseconfig.a")) + # suitesparse_libs = [os.path.join(suitesparse, x, "Lib", "lib%s.a" % x.replace("_", "").lower()) + # for x in ss_libs] + # suitesparse_libs.append(os.path.join(suitesparse, "SuiteSparse_config", "libsuitesparseconfig.a")) + suitesparse_libs = [os.path.join(suitesparse, "lib", "lib%s.so" % x.replace("_", "").lower()) for x in ss_libs] lib_spec = "-lib=[%s]" % ','.join(suitesparse_libs) else: # CHOLMOD and UMFPACK are part of SuiteSparse (PETSc < 3.5) diff --git a/easybuild/easyblocks/t/trilinos.py b/easybuild/easyblocks/t/trilinos.py index 030fbcd35b..73218c9513 100644 --- a/easybuild/easyblocks/t/trilinos.py +++ b/easybuild/easyblocks/t/trilinos.py @@ -31,7 +31,7 @@ import random import re -from easybuild.tools import LooseVersion +from distutils.version import LooseVersion import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.generic.cmakemake import CMakeMake @@ -144,14 +144,12 @@ def configure_step(self): if suitesparse: self.cfg.update('configopts', "-DTPL_ENABLE_UMFPACK:BOOL=ON") self.cfg.update('configopts', "-DTPL_ENABLE_Cholmod:BOOL=ON") - incdirs, libdirs, libnames = [], [], [] - for lib in ["UMFPACK", "CHOLMOD", "COLAMD", "AMD", "CCOLAMD", "CAMD"]: - incdirs.append(os.path.join(suitesparse, lib, "Include")) - libdirs.append(os.path.join(suitesparse, lib, "Lib")) - libnames.append(lib.lower()) + incdir = os.path.join(suitesparse, "include") + libdir = os.path.join(suitesparse, "lib") + libs = ["UMFPACK", "CHOLMOD", "COLAMD", "AMD", "CCOLAMD", "CAMD"] + libnames = [lib.lower() for lib in libs] # add SuiteSparse config lib, it is in recent versions of suitesparse - libdirs.append(os.path.join(suitesparse, 'SuiteSparse_config')) libnames.append('suitesparseconfig') # because of "SuiteSparse_config.c:function SuiteSparse_tic: error: undefined reference to 'clock_gettime'" libnames.append('rt') @@ -162,11 +160,11 @@ def configure_step(self): # see https://answers.launchpad.net/dorsal/+question/223167 libnames.append('libmetis.a') - self.cfg.update('configopts', '-DUMFPACK_INCLUDE_DIRS:PATH="%s"' % ';'.join(incdirs)) - self.cfg.update('configopts', '-DUMFPACK_LIBRARY_DIRS:PATH="%s"' % ';'.join(libdirs)) + self.cfg.update('configopts', '-DUMFPACK_INCLUDE_DIRS:PATH="%s"' % incdir) + self.cfg.update('configopts', '-DUMFPACK_LIBRARY_DIRS:PATH="%s"' % libdir) self.cfg.update('configopts', '-DUMFPACK_LIBRARY_NAMES:STRING="%s"' % ';'.join(libnames)) - self.cfg.update('configopts', '-DCholmod_INCLUDE_DIRS:PATH="%s"' % ';'.join(incdirs)) - self.cfg.update('configopts', '-DCholmod_LIBRARY_DIRS:PATH="%s"' % ';'.join(libdirs)) + self.cfg.update('configopts', '-DCholmod_INCLUDE_DIRS:PATH="%s"' % incdir) + self.cfg.update('configopts', '-DCholmod_LIBRARY_DIRS:PATH="%s"' % libdir) self.cfg.update('configopts', '-DCholmod_LIBRARY_NAMES:STRING="%s"' % ';'.join(libnames)) # BLACS From d3e9e53387eead6a87feefbe9a05a3860fe583a3 Mon Sep 17 00:00:00 2001 From: Chia-Jung Hsu Date: Mon, 18 Dec 2023 18:40:13 +0100 Subject: [PATCH 119/145] Remove commented line and fix linting --- easybuild/easyblocks/p/petsc.py | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/easybuild/easyblocks/p/petsc.py b/easybuild/easyblocks/p/petsc.py index 2b116fd447..fd802a8fba 100644 --- a/easybuild/easyblocks/p/petsc.py +++ b/easybuild/easyblocks/p/petsc.py @@ -254,33 +254,21 @@ def configure_step(self): ss_libs = ["UMFPACK", "KLU", "CHOLMOD", "BTF", "CCOLAMD", "COLAMD", "CAMD", "AMD"] # More libraries added after version 3.17 if LooseVersion(self.version) >= LooseVersion("3.17"): - # specified order of libs matters! - # ss_libs = ["UMFPACK", "KLU", "SPQR", "CHOLMOD", "BTF", "CCOLAMD", - # "COLAMD", "CSparse", "CXSparse", "LDL", "RBio", - # "SLIP_LU", "CAMD", "AMD"] - ss_libs = ["UMFPACK", "KLU", "SPQR", "CHOLMOD", "BTF", "CCOLAMD", - "COLAMD", "CXSparse", "LDL", "RBio", - "SLIP_LU", "CAMD", "AMD"] - + "COLAMD", "CXSparse", "LDL", "RBio", "SLIP_LU", "CAMD", "AMD"] - # suitesparse_inc = [os.path.join(suitesparse, x, "Include") - # for x in ss_libs] - # suitesparse_inc.append(os.path.join(suitesparse, "SuiteSparse_config")) - suitesparse_inc = [os.path.join(suitesparse, "include")] - inc_spec = "-include=[%s]" % ','.join(suitesparse_inc) + suitesparse_inc = os.path.join(suitesparse, "include") + inc_spec = "-include=[%s]" % suitesparse_inc - # suitesparse_libs = [os.path.join(suitesparse, x, "Lib", "lib%s.a" % x.replace("_", "").lower()) - # for x in ss_libs] - # suitesparse_libs.append(os.path.join(suitesparse, "SuiteSparse_config", "libsuitesparseconfig.a")) - suitesparse_libs = [os.path.join(suitesparse, "lib", "lib%s.so" % x.replace("_", "").lower()) for x in ss_libs] + suitesparse_libs = [os.path.join(suitesparse, "lib", "lib%s.so" % x.replace("_", "").lower()) + for x in ss_libs] lib_spec = "-lib=[%s]" % ','.join(suitesparse_libs) else: # CHOLMOD and UMFPACK are part of SuiteSparse (PETSc < 3.5) withdep = "--with-umfpack" - inc_spec = "-include=%s" % os.path.join(suitesparse, "UMFPACK", "Include") + inc_spec = "-include=%s" % os.path.join(suitesparse, "include") # specified order of libs matters! - umfpack_libs = [os.path.join(suitesparse, x, "Lib", "lib%s.a" % x.lower()) + umfpack_libs = [os.path.join(suitesparse, "lib", "lib%s.a" % x.lower()) for x in ["UMFPACK", "CHOLMOD", "COLAMD", "AMD"]] lib_spec = "-lib=[%s]" % ','.join(umfpack_libs) From 47f527102ad6ce8def3bc7b334f257259be923c9 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 20 Dec 2023 15:02:49 +0100 Subject: [PATCH 120/145] fix PythonPackage `det_pylibdir` for Python 3.12+ --- easybuild/easyblocks/generic/pythonpackage.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 63bb54b527..80efb8967a 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -172,8 +172,14 @@ def det_pylibdir(plat_specific=False, python_cmd=None): # determine Python lib dir via distutils # use run_cmd, we can to talk to the active Python, not the system Python running EasyBuild prefix = '/tmp/' - args = 'plat_specific=%s, prefix="%s"' % (plat_specific, prefix) - pycode = "import distutils.sysconfig; print(distutils.sysconfig.get_python_lib(%s))" % args + if LooseVersion(det_python_version(python_cmd)) >= LooseVersion('3.12'): + # Python 3.12 removed distutils but has a core sysconfig module which is similar + pathname = 'platlib' if plat_specific else 'purelib' + vars = {'platbase': prefix, 'base': prefix} + pycode = 'import sysconfig; sysconfig.get_path("%s", vars=%s)' % (pathname, vars) + else: + args = 'plat_specific=%s, prefix="%s"' % (plat_specific, prefix) + pycode = "import distutils.sysconfig; print(distutils.sysconfig.get_python_lib(%s))" % args cmd = "%s -c '%s'" % (python_cmd, pycode) log.debug("Determining Python library directory using command '%s'", cmd) From 1a343b77f3622a414754181e8e1d618a840bf2a8 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 20 Dec 2023 15:41:18 +0100 Subject: [PATCH 121/145] Add missing print and escapes --- easybuild/easyblocks/generic/pythonpackage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 80efb8967a..952666220c 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -176,11 +176,11 @@ def det_pylibdir(plat_specific=False, python_cmd=None): # Python 3.12 removed distutils but has a core sysconfig module which is similar pathname = 'platlib' if plat_specific else 'purelib' vars = {'platbase': prefix, 'base': prefix} - pycode = 'import sysconfig; sysconfig.get_path("%s", vars=%s)' % (pathname, vars) + pycode = 'import sysconfig; print(sysconfig.get_path("%s", vars=%s))' % (pathname, vars) else: args = 'plat_specific=%s, prefix="%s"' % (plat_specific, prefix) pycode = "import distutils.sysconfig; print(distutils.sysconfig.get_python_lib(%s))" % args - cmd = "%s -c '%s'" % (python_cmd, pycode) + cmd = "%s -c '%s'" % (python_cmd, pycode.replace("'", '"')) log.debug("Determining Python library directory using command '%s'", cmd) From b370d0050402923daedc8dd49d6ea475b9b7ea3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96hman?= Date: Wed, 20 Dec 2023 16:34:04 +0000 Subject: [PATCH 122/145] Add comment on why we disable check_for_sources --- easybuild/easyblocks/generic/cargopythonbundle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/cargopythonbundle.py b/easybuild/easyblocks/generic/cargopythonbundle.py index 50f0721671..ecaa58b7ee 100644 --- a/easybuild/easyblocks/generic/cargopythonbundle.py +++ b/easybuild/easyblocks/generic/cargopythonbundle.py @@ -50,7 +50,7 @@ def extra_options(extra_vars=None): def __init__(self, *args, **kwargs): """Constructor for CargoPythonBundle easyblock.""" - self.check_for_sources = False + self.check_for_sources = False # make Bundle allow sources (as crates are treated as sources) super(CargoPythonBundle, self).__init__(*args, **kwargs) def extract_step(self): From cc4bc7ff02507a171309db03f1f8b10b75addec7 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 21 Dec 2023 12:53:48 +0100 Subject: [PATCH 123/145] take into account that binutils might not be a dependency for TensorFlow E.g. when using `--filter-deps binutils` in combination with `--rpath` the could could fail to determine the files from binutils to collect as the "software root" won't be set. Fall back to collecting all rpath wrappers and a list of known binutils --- easybuild/easyblocks/t/tensorflow.py | 49 ++++++++++++++++++---------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/easybuild/easyblocks/t/tensorflow.py b/easybuild/easyblocks/t/tensorflow.py index 658dcca5e7..1993b574a4 100644 --- a/easybuild/easyblocks/t/tensorflow.py +++ b/easybuild/easyblocks/t/tensorflow.py @@ -35,6 +35,7 @@ import re import stat import tempfile +from itertools import chain import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain @@ -43,7 +44,7 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.tools import run, LooseVersion from easybuild.tools.build_log import EasyBuildError, print_warning -from easybuild.tools.config import build_option, IGNORE +from easybuild.tools.config import build_option, IGNORE, WARN, ERROR from easybuild.tools.filetools import adjust_permissions, apply_regex_substitutions, copy_file, mkdir, resolve_path from easybuild.tools.filetools import is_readable, read_file, symlink, which, write_file, remove_file from easybuild.tools.modules import get_software_root, get_software_version, get_software_libdir @@ -77,6 +78,8 @@ %(compiler_path)s "$@" """ +KNOWN_BINUTILS = ('ar', 'as', 'dwp', 'ld', 'ld.bfd', 'ld.gold', 'nm', 'objcopy', 'objdump', 'strip') + def split_tf_libs_txt(valid_libs_txt): """Split the VALID_LIBS entry from the TF file into single names""" @@ -300,7 +303,7 @@ def handle_jemalloc(self): def write_wrapper(self, wrapper_dir, compiler, i_mpi_root): """Helper function to write a compiler wrapper.""" wrapper_txt = INTEL_COMPILER_WRAPPER % { - 'compiler_path': which(compiler), + 'compiler_path': which(compiler, on_error=IGNORE if self.dry_run else ERROR), 'intel_mpi_root': i_mpi_root, 'cpath': os.getenv('CPATH'), 'intel_license_file': os.getenv('INTEL_LICENSE_FILE', os.getenv('LM_LICENSE_FILE')), @@ -463,20 +466,33 @@ def configure_step(self): # determine location where binutils' ld command is installed # note that this may be an RPATH wrapper script (when EasyBuild is configured with --rpath) - ld_path = which('ld') + ld_path = which('ld', on_error=ERROR) self.binutils_bin_path = os.path.dirname(ld_path) if self.toolchain.is_rpath_wrapper(ld_path): - if os.path.basename(os.path.dirname(os.path.dirname(ld_path))) == RPATH_WRAPPERS_SUBDIR: - # TF expects all binutils binaries in a single path but newer EB puts each in their own subfolder + # TF expects all binutils binaries in a single path but newer EB puts each in its own subfolder + # This new layout is: /RPATH_WRAPPERS_SUBDIR/_folder/ + rpath_wrapper_root = os.path.dirname(os.path.dirname(ld_path)) + if os.path.basename(rpath_wrapper_root) == RPATH_WRAPPERS_SUBDIR: # Add symlinks to each binutils binary into a single folder new_rpath_wrapper_dir = os.path.join(self.wrapper_dir, RPATH_WRAPPERS_SUBDIR) - binutils_bin_path = os.path.join(get_software_root('binutils'), 'bin') - self.log.info("Found %s to be an rpath wrapper. Adding symlinks for binutils in %s to %s.", - ld_path, binutils_bin_path, new_rpath_wrapper_dir) + binutils_root = get_software_root('binutils') + if binutils_root: + self.log.debug("Using binutils dependency at %s to gather binutils files.", binutils_root) + binutils_files = next(os.walk(os.path.join(binutils_root, 'bin')))[2] + else: + # binutils might be filtered (--filter-deps), so recursively gather files in the rpath wrapper dir + binutils_files = {f for (_, _, files) in os.walk(rpath_wrapper_root) for f in files} + # And add known ones + binutils_files.update(KNOWN_BINUTILS) + self.log.info("Found %s to be an rpath wrapper. Adding symlinks for binutils (%s) to %s.", + ld_path, ', '.join(binutils_files), new_rpath_wrapper_dir) mkdir(new_rpath_wrapper_dir) - for file in next(os.walk(binutils_bin_path))[2]: - # Note the use of `which` to take rpath wrappers where available - symlink(which(file), os.path.join(new_rpath_wrapper_dir, file)) + for file in binutils_files: + # use `which` to take rpath wrappers where available + # Ignore missing ones if bintuils was filtered (in which case we used a heuristic) + path = which(file, on_error=ERROR if binutils_root else WARN) + if path: + symlink(path, os.path.join(new_rpath_wrapper_dir, file)) self.binutils_bin_path = new_rpath_wrapper_dir # filter out paths from CPATH and LIBRARY_PATH. This is needed since bazel will pull some dependencies that @@ -586,9 +602,9 @@ def configure_step(self): # $GCC_HOST_COMPILER_PATH should be set to path of the actual compiler (not the MPI compiler wrapper) if use_mpi: - compiler_path = which(os.getenv('CC_SEQ')) + compiler_path = which(os.getenv('CC_SEQ'), on_error=ERROR) else: - compiler_path = which(os.getenv('CC')) + compiler_path = which(os.getenv('CC'), on_error=ERROR) # list of CUDA compute capabilities to use can be specifed in two ways (where (2) overrules (1)): # (1) in the easyconfig file, via the custom cuda_compute_capabilities; @@ -764,12 +780,11 @@ def patch_crosstool_files(self): (r'(cxx_builtin_include_directory:).*', ''), (r'^toolchain {', 'toolchain {\n' + '\n'.join(cxx_inc_dirs)), ] - for tool in ['ar', 'cpp', 'dwp', 'gcc', 'gcov', 'ld', 'nm', 'objcopy', 'objdump', 'strip']: - path = which(tool) + required_tools = {'ar', 'cpp', 'dwp', 'gcc', 'gcov', 'ld', 'nm', 'objcopy', 'objdump', 'strip'} + for tool in set(chain(required_tools, KNOWN_BINUTILS)): + path = which(tool, on_error=ERROR if tool in required_tools else WARN) if path: regex_subs.append((os.path.join('/usr', 'bin', tool), path)) - else: - raise EasyBuildError("Failed to determine path to '%s'", tool) # -fPIE/-pie and -fPIC are not compatible, so patch out hardcoded occurences of -fPIE/-pie if -fPIC is used if self.toolchain.options.get('pic', None): From beb5bd466fba85ed042f04080c875abf7f69844e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 21 Dec 2023 14:22:10 +0100 Subject: [PATCH 124/145] fix typo in comment in TensorFlow easyblock --- easybuild/easyblocks/t/tensorflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/t/tensorflow.py b/easybuild/easyblocks/t/tensorflow.py index 1993b574a4..174a5e38fe 100644 --- a/easybuild/easyblocks/t/tensorflow.py +++ b/easybuild/easyblocks/t/tensorflow.py @@ -489,7 +489,7 @@ def configure_step(self): mkdir(new_rpath_wrapper_dir) for file in binutils_files: # use `which` to take rpath wrappers where available - # Ignore missing ones if bintuils was filtered (in which case we used a heuristic) + # Ignore missing ones if binutils was filtered (in which case we used a heuristic) path = which(file, on_error=ERROR if binutils_root else WARN) if path: symlink(path, os.path.join(new_rpath_wrapper_dir, file)) From 619920c603ad5485f13adc22022c23b0436a38f9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 22 Dec 2023 11:32:54 +0100 Subject: [PATCH 125/145] fix description for CargoPythonBundle --- easybuild/easyblocks/generic/cargopythonbundle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/cargopythonbundle.py b/easybuild/easyblocks/generic/cargopythonbundle.py index ecaa58b7ee..4745b8a91e 100644 --- a/easybuild/easyblocks/generic/cargopythonbundle.py +++ b/easybuild/easyblocks/generic/cargopythonbundle.py @@ -23,7 +23,7 @@ # along with EasyBuild. If not, see . ## """ -EasyBuild support for installing python bundles with some packages using Rust +EasyBuild support for installing a bundle of Python packages, where some are built with Rust @author: Mikael Oehman (Chalmers University of Technology) """ From dafa653c4691bb6e3c3b9b4cca03e82257ff5aeb Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 22 Dec 2023 11:52:14 +0100 Subject: [PATCH 126/145] add docstring for IntelBase.set_versioned_subdir --- easybuild/easyblocks/generic/intelbase.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/easyblocks/generic/intelbase.py b/easybuild/easyblocks/generic/intelbase.py index 31f44d1935..cb59b69b67 100644 --- a/easybuild/easyblocks/generic/intelbase.py +++ b/easybuild/easyblocks/generic/intelbase.py @@ -129,6 +129,7 @@ def get_versioned_subdir(self, subdir): return self._latest_subdir[subdir] def set_versioned_subdir(self, subdir, path): + """Set version-specific path for specified subdirectory.""" self._latest_subdir[subdir] = path def get_guesses_tools(self): From 58c22a0fea4e85e2b0630f489b344fb557ce8ae7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 22 Dec 2023 11:53:36 +0100 Subject: [PATCH 127/145] rename `builderdir` local variable in `build_mkl_flexiblas` to `builder_subdir` in --- easybuild/easyblocks/i/imkl.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/i/imkl.py b/easybuild/easyblocks/i/imkl.py index 883b773535..d04f59c4e0 100644 --- a/easybuild/easyblocks/i/imkl.py +++ b/easybuild/easyblocks/i/imkl.py @@ -321,10 +321,10 @@ def build_mkl_flexiblas(self, flexiblasdir): and libflexiblas_imkl_sequential.so. They can be used as FlexiBLAS backends via FLEXIBLAS_LIBRARY_PATH. """ - builderdir = os.path.join('tools', 'builder') + builder_subdir = os.path.join('tools', 'builder') if LooseVersion(self.version) >= LooseVersion('2024'): - builderdir = os.path.join('share', 'mkl', builderdir) - change_dir(os.path.join(self.installdir, self.mkl_basedir, builderdir)) + builder_subdir = os.path.join('share', 'mkl', builder_subdir) + change_dir(os.path.join(self.installdir, self.mkl_basedir, builder_subdir)) mkdir(flexiblasdir, parents=True) # concatenate lists of all BLAS, CBLAS and LAPACK functions From a0ccc4e86caba908725053d876afbf2492926b35 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 28 Dec 2023 20:47:30 +0100 Subject: [PATCH 128/145] fix nvptx sanity check for Clang >= 14.x --- easybuild/easyblocks/c/clang.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/easybuild/easyblocks/c/clang.py b/easybuild/easyblocks/c/clang.py index f66ddc5dde..3423e1c96b 100644 --- a/easybuild/easyblocks/c/clang.py +++ b/easybuild/easyblocks/c/clang.py @@ -622,9 +622,11 @@ def sanity_check_step(self): custom_commands = ['clang --help', 'clang++ --help', 'llvm-config --cxxflags'] shlib_ext = get_shared_lib_ext() + version = LooseVersion(self.version) + # Clang v16+ only use the major version number for the resource dir resdir_version = self.version - if LooseVersion(self.version) >= LooseVersion('16'): + if version >= '16': resdir_version = self.version.split('.')[0] # Detect OpenMP support for CPU architecture @@ -640,7 +642,7 @@ def sanity_check_step(self): else: print_warning("Unknown CPU architecture (%s) for OpenMP and runtime libraries check!" % arch) - if LooseVersion(self.version) >= LooseVersion('14'): + if version >= '14': glob_pattern = os.path.join(self.installdir, 'lib', '%s-*' % arch) matches = glob.glob(glob_pattern) if matches: @@ -663,7 +665,7 @@ def sanity_check_step(self): if self.cfg['static_analyzer']: custom_paths['files'].extend(["bin/scan-build", "bin/scan-view"]) - if 'clang-tools-extra' in self.cfg['llvm_projects'] and LooseVersion(self.version) >= LooseVersion('3.4'): + if 'clang-tools-extra' in self.cfg['llvm_projects'] and version >= '3.4': custom_paths['files'].extend(["bin/clang-tidy"]) if 'polly' in self.cfg['llvm_projects']: @@ -685,15 +687,15 @@ def sanity_check_step(self): if 'libcxxabi' in self.cfg['llvm_runtimes']: custom_paths['files'].extend([os.path.join(self.runtime_lib_path, "libc++abi.%s" % shlib_ext)]) - if 'flang' in self.cfg['llvm_projects'] and LooseVersion(self.version) >= LooseVersion('15'): + if 'flang' in self.cfg['llvm_projects'] and version >= '15': flang_compiler = 'flang-new' custom_paths['files'].extend(["bin/%s" % flang_compiler]) custom_commands.extend(["%s --help" % flang_compiler]) - if LooseVersion(self.version) >= LooseVersion('3.8'): + if version >= '3.8': custom_paths['files'].extend(["lib/libomp.%s" % shlib_ext, "lib/clang/%s/include/omp.h" % resdir_version]) - if LooseVersion(self.version) >= LooseVersion('12'): + if version >= '12': omp_target_libs = ["lib/libomptarget.%s" % shlib_ext, "lib/libomptarget.rtl.%s.%s" % (arch, shlib_ext)] else: omp_target_libs = ["lib/libomptarget.%s" % shlib_ext] @@ -703,24 +705,26 @@ def sanity_check_step(self): if 'NVPTX' in self.cfg['build_targets']: custom_paths['files'].append("lib/libomptarget.rtl.cuda.%s" % shlib_ext) # The static 'nvptx.a' library is not built from version 12 onwards - if LooseVersion(self.version) < LooseVersion('12.0'): + if version < '12.0': custom_paths['files'].append("lib/libomptarget-nvptx.a") ec_cuda_cc = self.cfg['cuda_compute_capabilities'] cfg_cuda_cc = build_option('cuda_compute_capabilities') cuda_cc = cfg_cuda_cc or ec_cuda_cc or [] # We need the CUDA capability in the form of '75' and not '7.5' cuda_cc = [cc.replace('.', '') for cc in cuda_cc] - if LooseVersion('12.0') < LooseVersion(self.version) < LooseVersion('13.0'): + if '12.0' < version < '13.0': custom_paths['files'].extend(["lib/libomptarget-nvptx-cuda_%s-sm_%s.bc" % (x, y) for x in CUDA_TOOLKIT_SUPPORT for y in cuda_cc]) - else: + # libomptarget-nvptx-sm*.bc is not there for Clang 14.x; + elif version < '14.0' or version >= '15.0': custom_paths['files'].extend(["lib/libomptarget-nvptx-sm_%s.bc" % cc for cc in cuda_cc]) # From version 13, and hopefully onwards, the naming of the CUDA # '.bc' files became a bit simpler and now we don't need to take # into account the CUDA version Clang was compiled with, making it - # easier to check for the bitcode files we expect - if LooseVersion(self.version) >= LooseVersion('13.0'): + # easier to check for the bitcode files we expect; + # libomptarget-new-nvptx-sm*.bc is only there in Clang 13.x and 14.x; + if version >= '13.0' and version < '15.0': custom_paths['files'].extend(["lib/libomptarget-new-nvptx-sm_%s.bc" % cc for cc in cuda_cc]) # If building for AMDGPU check that OpenMP target library was created @@ -729,12 +733,12 @@ def sanity_check_step(self): # OpenMP offloading support to AMDGPU was not added until version # 13, however, building for the AMDGPU target predates this and so # doesn't necessarily mean that the AMDGPU target failed - if LooseVersion(self.version) >= LooseVersion('13.0'): + if version >= '13.0': custom_paths['files'].append("lib/libomptarget.rtl.amdgpu.%s" % shlib_ext) custom_paths['files'].extend(["lib/libomptarget-amdgcn-%s.bc" % gfx for gfx in self.cfg['amd_gfx_list']]) custom_paths['files'].append("bin/amdgpu-arch") - if LooseVersion(self.version) >= LooseVersion('14.0'): + if version >= '14.0': custom_paths['files'].extend(["lib/libomptarget-new-amdgpu-%s.bc" % gfx for gfx in self.cfg['amd_gfx_list']]) From e3bbe0b4a467a5285a95f8556da4d911feee9831 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 29 Dec 2023 10:26:40 +0100 Subject: [PATCH 129/145] consider both $EB_COMSOL_LICENSE_FILE and $LMCOMSOL_LICENSE_FILE in COMSOL easyblock --- easybuild/easyblocks/c/comsol.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/easybuild/easyblocks/c/comsol.py b/easybuild/easyblocks/c/comsol.py index 6be902ae17..dd69c2dac3 100644 --- a/easybuild/easyblocks/c/comsol.py +++ b/easybuild/easyblocks/c/comsol.py @@ -59,14 +59,14 @@ def extract_step(self): def configure_step(self): """Configure COMSOL installation: create license file.""" - default_lic_env_var = 'EB_COMSOL_LICENSE_FILE' - lic_specs, self.license_env_var = find_flexlm_license(custom_env_vars=[default_lic_env_var], + comsol_lic_env_vars = ['EB_COMSOL_LICENSE_FILE', 'LMCOMSOL_LICENSE_FILE'] + lic_specs, self.license_env_var = find_flexlm_license(custom_env_vars=comsol_lic_env_vars, lic_specs=[self.cfg['license_file']]) if lic_specs: if self.license_env_var is None: self.log.info("Using COMSOL license specifications from 'license_file': %s", lic_specs) - self.license_env_var = default_lic_env_var + self.license_env_var = comsol_lic_env_vars[0] else: self.log.info("Using COMSOL license specifications from $%s: %s", self.license_env_var, lic_specs) @@ -74,7 +74,7 @@ def configure_step(self): env.setvar(self.license_env_var, self.license_file) else: msg = "No viable license specifications found; " - msg += "specify 'license_file', or define $%s" % default_lic_env_var + msg += "specify 'license_file', or define %s" % (', '.join('$%s' % x for x in comsol_lic_env_vars)) raise EasyBuildError(msg) copy_file(os.path.join(self.start_dir, 'setupconfig.ini'), self.configfile) From 66e2ba87bbbf29051162e5217af5e2051a4f1dd2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 29 Dec 2023 11:05:48 +0100 Subject: [PATCH 130/145] import LooseVersion from easybuild.tools in PETSc + Trilinos easyblocks --- easybuild/easyblocks/p/petsc.py | 2 +- easybuild/easyblocks/t/trilinos.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/p/petsc.py b/easybuild/easyblocks/p/petsc.py index fd802a8fba..d5081d528a 100644 --- a/easybuild/easyblocks/p/petsc.py +++ b/easybuild/easyblocks/p/petsc.py @@ -29,12 +29,12 @@ """ import os import re -from distutils.version import LooseVersion import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.framework.easyconfig import BUILD, CUSTOM +from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import symlink, apply_regex_substitutions from easybuild.tools.modules import get_software_root, get_software_version diff --git a/easybuild/easyblocks/t/trilinos.py b/easybuild/easyblocks/t/trilinos.py index 73218c9513..51ed37e9cc 100644 --- a/easybuild/easyblocks/t/trilinos.py +++ b/easybuild/easyblocks/t/trilinos.py @@ -31,11 +31,10 @@ import random import re -from distutils.version import LooseVersion - import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.generic.cmakemake import CMakeMake from easybuild.framework.easyconfig import CUSTOM +from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_path from easybuild.tools.filetools import mkdir, remove_dir, symlink From 168146da156c2f79ad09a0b66b1f4259422524f8 Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Fri, 29 Dec 2023 20:28:48 +0100 Subject: [PATCH 131/145] prepare release notes for EasyBuild v4.9.0 + bump version to 4.9.0 --- RELEASE_NOTES | 35 +++++++++++++++++++++++++++++++- easybuild/easyblocks/__init__.py | 2 +- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 027e4a47e3..06eea70afa 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -3,7 +3,40 @@ For more detailed information, please see the git log. These release notes can also be consulted at http://easybuild.readthedocs.org/en/latest/Release_notes.html. -The latest version of easybuild-easyblocks provides 251 software-specific easyblocks and 42 generic easyblocks. +The latest version of easybuild-easyblocks provides 254 software-specific easyblocks and 43 generic easyblocks. + + +v4.9.0 (30 December 2023) +------------------------- + +feature release + +- add generic `CargoPythonBundle` easyblock (#2964) +- add new easyblock for HPCC and adapt HPL easyblock (#3009) +- new easyblock for PALM (#3020) +- add custom easyblock for flook that includes install step (#3034) +- minor enhancements and updates, including: + - add custom easyconfig parameter `cmake_options` to SuiteSparse easyblock (#3031) + - update custom intel-compilers easyblock for versions >= 2024 (#3037) + - update custom easyblock for Intel MPI easyblock for v2021.11 (#3039) + - update numpy easyblock for v1.26+ (#3041) + - update custom easyblock for Intel MKL for v2024.x (#3042) + - update Ferret easyblock to be compatible with v7.6.0 (#3052) +- various bug fixes, including: + - add support for allowing version mismatch + consider versionsuffix when creating `.modulerc` in `ModuleRC` easyblock (#3028) + - update error detection for PyTorch tests (#3033) + - disable LLVM build downloads from CI in Rust (#3038) + - add requirement for EULA acceptance to CUDA easyblock (#3045) + - make various fixes and enhancements to NWChem easyblock (#3049) + - Add binutils symlinks when building TensorFlow with `--rpath` (#3054) + - fix specifying path to SuiteSparse header files and libraries in numpy, Trilinos, PETSc easyblocks (#3056) + - fix `det_pylibdir` provided by `PythonPackage` easyblock for Python 3.12+ (#3057) + - take into account that binutils might be a filtered dependency for TensorFlow (#3058) + - fix nvptx sanity check for Clang >= 14.x (#3059) +- other changes: + - update SuiteSparse easyblock to only install SuiteSparse libraries with `make install` (#3004) + - also consider `$EB_COMSOL_LICENSE_FILE` environment variable in custom easyblock for COMSOL (#3044) + - import `LooseVersion` from `easybuild.tools` instead of `distutils.version` in easyblocks (#3048) v4.8.2 (29 October 2023) diff --git a/easybuild/easyblocks/__init__.py b/easybuild/easyblocks/__init__.py index 115bad70c1..80b10f5187 100644 --- a/easybuild/easyblocks/__init__.py +++ b/easybuild/easyblocks/__init__.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.9.0.dev0') +VERSION = LooseVersion('4.9.0') UNKNOWN = 'UNKNOWN' From 00ca1e7e0a13dbd81ee57a5a890e2d008400c5db Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 30 Dec 2023 09:17:34 +0100 Subject: [PATCH 132/145] minor tweaks in v4.9.0 release notes --- RELEASE_NOTES | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 06eea70afa..6dfb5b5672 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -12,9 +12,7 @@ v4.9.0 (30 December 2023) feature release - add generic `CargoPythonBundle` easyblock (#2964) -- add new easyblock for HPCC and adapt HPL easyblock (#3009) -- new easyblock for PALM (#3020) -- add custom easyblock for flook that includes install step (#3034) +- 3 new software-specific easyblocks: flook (#3034), HPCC (#3009), PALM (#3020) - minor enhancements and updates, including: - add custom easyconfig parameter `cmake_options` to SuiteSparse easyblock (#3031) - update custom intel-compilers easyblock for versions >= 2024 (#3037) @@ -28,10 +26,9 @@ feature release - disable LLVM build downloads from CI in Rust (#3038) - add requirement for EULA acceptance to CUDA easyblock (#3045) - make various fixes and enhancements to NWChem easyblock (#3049) - - Add binutils symlinks when building TensorFlow with `--rpath` (#3054) + - add binutils symlinks when building TensorFlow with `--rpath` (#3054, #3058) - fix specifying path to SuiteSparse header files and libraries in numpy, Trilinos, PETSc easyblocks (#3056) - fix `det_pylibdir` provided by `PythonPackage` easyblock for Python 3.12+ (#3057) - - take into account that binutils might be a filtered dependency for TensorFlow (#3058) - fix nvptx sanity check for Clang >= 14.x (#3059) - other changes: - update SuiteSparse easyblock to only install SuiteSparse libraries with `make install` (#3004) From ba3635f69cb2680323ccbb3cc6a36129b487f72c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 30 Dec 2023 21:16:00 +0100 Subject: [PATCH 133/145] bump version to 4.9.1dev --- easybuild/easyblocks/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/__init__.py b/easybuild/easyblocks/__init__.py index 80b10f5187..ab8aeea30f 100644 --- a/easybuild/easyblocks/__init__.py +++ b/easybuild/easyblocks/__init__.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.9.0') +VERSION = LooseVersion('4.9.1.dev0') UNKNOWN = 'UNKNOWN' From bd7ab048fd9ae4303a8bb8e8be1707f43694dac4 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Wed, 3 Jan 2024 02:24:32 +0800 Subject: [PATCH 134/145] fix ELPA preprocessor flag for QuantumESPRESSO >= 7 --- easybuild/easyblocks/q/quantumespresso.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/q/quantumespresso.py b/easybuild/easyblocks/q/quantumespresso.py index 15d00f53d0..de3ba33719 100644 --- a/easybuild/easyblocks/q/quantumespresso.py +++ b/easybuild/easyblocks/q/quantumespresso.py @@ -133,9 +133,22 @@ def configure_step(self): raise EasyBuildError("ELPA requires ScaLAPACK but 'with_scalapack' is set to False") elpa_v = get_software_version("ELPA") - if LooseVersion(self.version) >= LooseVersion("6"): + if LooseVersion(self.version) >= LooseVersion("7"): + # NOTE: from version 7, there are only three __ELPA flags, + # - __ELPA for ELPA releases 2018.11 and beyond; + # - __ELPA_2016 for ELPA releases 2016.11, 2017.x and 2018.05; + # - __ELPA_2015 for ELPA releases 2015.x and 2016.05; + # see https://github.com/QEF/q-e/commit/351f4871fee3c8045d75592dde606b2279b08e02 + if LooseVersion(elpa_v) >= LooseVersion("2018.11"): + dflags.append('-D__ELPA') + elif LooseVersion(elpa_v) >= LooseVersion("2016.11"): + dflags.append('-D__ELPA_2016') + else: + dflags.append('-D__ELPA_2015') - # NOTE: Quantum Espresso should use -D__ELPA_ for corresponding ELPA version + elpa_min_ver = "2015" + elif LooseVersion(self.version) >= LooseVersion("6"): + # NOTE: Quantum Espresso 6.x should use -D__ELPA_ for corresponding ELPA version # However for ELPA VERSION >= 2017.11 Quantum Espresso needs to use ELPA_2018 # because of outdated bindings. See: https://xconfigure.readthedocs.io/en/latest/elpa/ if LooseVersion("2018") > LooseVersion(elpa_v) >= LooseVersion("2017.11"): From 7f9d76990088f4e634581b1ed83c115bd78b5425 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 3 Jan 2024 17:12:05 +0100 Subject: [PATCH 135/145] Show build output of PyTorch when using pip --- easybuild/easyblocks/p/pytorch.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easybuild/easyblocks/p/pytorch.py b/easybuild/easyblocks/p/pytorch.py index ec9b40a5ee..3d5cc8e93f 100644 --- a/easybuild/easyblocks/p/pytorch.py +++ b/easybuild/easyblocks/p/pytorch.py @@ -243,6 +243,10 @@ def configure_step(self): self.cfg.update('prebuildopts', ' '.join(unique_options) + ' ') self.cfg.update('preinstallopts', ' '.join(unique_options) + ' ') + if self.cfg['use_pip']: + # Make pip show output of build process + self.cfg.update('installopts', '--verbose') + def _set_cache_dir(self): """Set $XDG_CACHE_HOME to avoid PyTorch defaulting to $HOME""" cache_dir = os.path.join(self.tmpdir, '.cache') From 03984c356c98b922939585e0ed6001e9d6a9dac7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 4 Jan 2024 09:27:27 +0100 Subject: [PATCH 136/145] remove custom easyblock for EggLib --- easybuild/easyblocks/e/egglib.py | 88 -------------------------------- 1 file changed, 88 deletions(-) delete mode 100644 easybuild/easyblocks/e/egglib.py diff --git a/easybuild/easyblocks/e/egglib.py b/easybuild/easyblocks/e/egglib.py deleted file mode 100644 index f4aa262042..0000000000 --- a/easybuild/easyblocks/e/egglib.py +++ /dev/null @@ -1,88 +0,0 @@ -## -# Copyright 2009-2023 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# https://github.com/easybuilders/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -EasyBuild support for building and installing EggLib, implemented as an easyblock - -@author: Kenneth Hoste (HPC-UGent) -""" -import os - -import easybuild.tools.environment as env -from easybuild.easyblocks.generic.configuremake import ConfigureMake -from easybuild.easyblocks.generic.pythonpackage import PythonPackage -from easybuild.tools.build_log import EasyBuildError - - -class EB_EggLib(PythonPackage, ConfigureMake): - """Support for building/installing EggLib.""" - - def configure_step(self): - """Configure EggLib build/install procedure.""" - # only need to configure Python library here, configuration of C++ library is done in install step - PythonPackage.configure_step(self) - - def build_step(self): - """No custom build procedure for EggLib; build/install is done in install_step.""" - pass - - def install_step(self): - """Custom install procedure for EggLib: first build/install C++ library, then build Python library.""" - - # build/install C++ library - cpp_subdir = os.path.join(self.builddir, 'egglib-cpp-%s' % self.version) - try: - os.chdir(cpp_subdir) - except OSError as err: - raise EasyBuildError("Failed to move to: %s", err) - - ConfigureMake.configure_step(self) - ConfigureMake.build_step(self) - ConfigureMake.install_step(self) - - # header files and libraries must be found when building Python library - for varname, subdir in [('CPATH', 'include'), ('LIBRARY_PATH', 'lib')]: - env.setvar(varname, '%s:%s' % (os.path.join(self.installdir, subdir), os.environ.get(varname, ''))) - - # build/install Python package - py_subdir = os.path.join(self.builddir, 'egglib-py-%s' % self.version) - try: - os.chdir(py_subdir) - except OSError as err: - raise EasyBuildError("Failed to move to: %s", err) - - PythonPackage.build_step(self) - - self.cfg.update('installopts', "--install-lib %s" % os.path.join(self.installdir, self.pylibdir)) - self.cfg.update('installopts', "--install-scripts %s" % os.path.join(self.installdir, 'bin')) - - PythonPackage.install_step(self) - - def sanity_check_step(self): - """Custom sanity check for EggLib.""" - custom_paths = { - 'files': ['bin/egglib', 'lib/libegglib-cpp.a'], - 'dirs': ['include/egglib-cpp', self.pylibdir], - } - super(EB_EggLib, self).sanity_check_step(custom_paths=custom_paths) From c04cbaed123a42677cf6cdc40fafa02554f329e9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 4 Jan 2024 09:32:58 +0100 Subject: [PATCH 137/145] remove unused 'stat' in SuiteSparse easyblock --- easybuild/easyblocks/s/suitesparse.py | 1 - 1 file changed, 1 deletion(-) diff --git a/easybuild/easyblocks/s/suitesparse.py b/easybuild/easyblocks/s/suitesparse.py index f01f76b752..c581b3c7a1 100644 --- a/easybuild/easyblocks/s/suitesparse.py +++ b/easybuild/easyblocks/s/suitesparse.py @@ -35,7 +35,6 @@ import fileinput import re import os -import stat import sys from easybuild.tools import LooseVersion From d3e316a39023d5ceb2a5bb8edec0cad9a9e4aa0d Mon Sep 17 00:00:00 2001 From: Jasper Grimm <65227842+jfgrimm@users.noreply.github.com> Date: Thu, 4 Jan 2024 15:51:51 +0000 Subject: [PATCH 138/145] use super --- easybuild/easyblocks/d/dualsphysics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/d/dualsphysics.py b/easybuild/easyblocks/d/dualsphysics.py index f83bd010a4..1e83891ccb 100644 --- a/easybuild/easyblocks/d/dualsphysics.py +++ b/easybuild/easyblocks/d/dualsphysics.py @@ -75,7 +75,7 @@ def prepare_step(self, *args, **kwargs): def configure_step(self): """Custom configure procedure for DualSPHysics.""" srcdir = os.path.join(self.cfg['start_dir'], 'src/source') - CMakeMakeCp.configure_step(self, srcdir=srcdir) + super(EB_DUALSPHysics, self).configure_step(srcdir=srcdir) def install_step(self): """Custom install procedure for DualSPHysics.""" From 9d0d75cb9b93fc39cf40aa7a876c97294570e800 Mon Sep 17 00:00:00 2001 From: Jasper Grimm <65227842+jfgrimm@users.noreply.github.com> Date: Thu, 4 Jan 2024 15:53:23 +0000 Subject: [PATCH 139/145] fix typo --- easybuild/easyblocks/d/dualsphysics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/d/dualsphysics.py b/easybuild/easyblocks/d/dualsphysics.py index 1e83891ccb..cde522aa80 100644 --- a/easybuild/easyblocks/d/dualsphysics.py +++ b/easybuild/easyblocks/d/dualsphysics.py @@ -75,7 +75,7 @@ def prepare_step(self, *args, **kwargs): def configure_step(self): """Custom configure procedure for DualSPHysics.""" srcdir = os.path.join(self.cfg['start_dir'], 'src/source') - super(EB_DUALSPHysics, self).configure_step(srcdir=srcdir) + super(EB_DualSPHysics, self).configure_step(srcdir=srcdir) def install_step(self): """Custom install procedure for DualSPHysics.""" From 87fd8bcdd01db7d45d2306e3676668fb36fce16d Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 5 Jan 2024 09:01:03 +0100 Subject: [PATCH 140/145] Introduce `pip_verbose` for PythonPackage and use in in PyTorch --- easybuild/easyblocks/generic/pythonpackage.py | 5 ++++- easybuild/easyblocks/p/pytorch.py | 6 ++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 952666220c..08a1b49632 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -344,6 +344,8 @@ def extra_options(extra_vars=None): 'pip_ignore_installed': [True, "Let pip ignore installed Python packages (i.e. don't remove them)", CUSTOM], 'pip_no_index': [None, "Pass --no-index to pip to disable connecting to PyPi entirely which also disables " "the pip version check. Enabled by default when pip_ignore_installed=True", CUSTOM], + 'pip_verbose': [None, "Pass --verbose to 'pip install' (if pip is used). " + "Defaults to 'True' if the EB option --debug is used.", CUSTOM], 'req_py_majver': [None, "Required major Python version (only relevant when using system Python)", CUSTOM], 'req_py_minver': [None, "Required minor Python version (only relevant when using system Python)", CUSTOM], 'sanity_pip_check': [False, "Run 'python -m pip check' to ensure all required Python packages are " @@ -427,7 +429,8 @@ def determine_install_command(self): if self.cfg.get('use_pip', False) or self.cfg.get('use_pip_editable', False): self.install_cmd = PIP_INSTALL_CMD - if build_option('debug'): + pip_verbose = self.cfg.get('pip_verbose', None) + if pip_verbose or (pip_verbose is None and build_option('debug')): self.cfg.update('installopts', '--verbose') # don't auto-install dependencies with pip unless use_pip_for_deps=True diff --git a/easybuild/easyblocks/p/pytorch.py b/easybuild/easyblocks/p/pytorch.py index 3d5cc8e93f..62172fca10 100644 --- a/easybuild/easyblocks/p/pytorch.py +++ b/easybuild/easyblocks/p/pytorch.py @@ -56,6 +56,8 @@ def extra_options(): }) extra_vars['download_dep_fail'][0] = True extra_vars['sanity_pip_check'][0] = True + # Make pip show output of build process as that may often contain errors or important warnings + extra_vars['pip_verbose'][0] = True return extra_vars @@ -243,10 +245,6 @@ def configure_step(self): self.cfg.update('prebuildopts', ' '.join(unique_options) + ' ') self.cfg.update('preinstallopts', ' '.join(unique_options) + ' ') - if self.cfg['use_pip']: - # Make pip show output of build process - self.cfg.update('installopts', '--verbose') - def _set_cache_dir(self): """Set $XDG_CACHE_HOME to avoid PyTorch defaulting to $HOME""" cache_dir = os.path.join(self.tmpdir, '.cache') From aed6f676abb7fd79e88d8e56c9f01c5c1b58603c Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Fri, 5 Jan 2024 09:28:42 +0000 Subject: [PATCH 141/145] bypass .mod file in Clang, AOCC, AOMP, and CPLEX --- easybuild/easyblocks/a/aocc.py | 3 +++ easybuild/easyblocks/a/aomp.py | 2 ++ easybuild/easyblocks/c/clang.py | 3 +++ easybuild/easyblocks/c/cplex.py | 3 +++ 4 files changed, 11 insertions(+) diff --git a/easybuild/easyblocks/a/aocc.py b/easybuild/easyblocks/a/aocc.py index aff71ee1b0..5cc87a27e7 100644 --- a/easybuild/easyblocks/a/aocc.py +++ b/easybuild/easyblocks/a/aocc.py @@ -67,6 +67,9 @@ def __init__(self, *args, **kwargs): self.clangversion = self.cfg['clangversion'] + # Bypass the .mod file check for GCCcore installs + self.cfg['skip_mod_files_sanity_check'] = True + def _aocc_guess_clang_version(self): map_aocc_to_clang_ver = { '2.3.0': '11.0.0', diff --git a/easybuild/easyblocks/a/aomp.py b/easybuild/easyblocks/a/aomp.py index 7ff4126d6d..f0aaae4138 100644 --- a/easybuild/easyblocks/a/aomp.py +++ b/easybuild/easyblocks/a/aomp.py @@ -68,6 +68,8 @@ def __init__(self, *args, **kwargs): super(EB_AOMP, self).__init__(*args, **kwargs) self.cfg['extract_sources'] = True self.cfg['dontcreateinstalldir'] = True + # Bypass the .mod file check for GCCcore installs + self.cfg['skip_mod_files_sanity_check'] = True def configure_step(self): """Configure AOMP build and let 'Binary' install""" diff --git a/easybuild/easyblocks/c/clang.py b/easybuild/easyblocks/c/clang.py index 3423e1c96b..4411b34e4c 100644 --- a/easybuild/easyblocks/c/clang.py +++ b/easybuild/easyblocks/c/clang.py @@ -126,6 +126,9 @@ def __init__(self, *args, **kwargs): self.make_parallel_opts = "" self.runtime_lib_path = "lib" + # Bypass the .mod file check for GCCcore installs + self.cfg['skip_mod_files_sanity_check'] = True + if not self.cfg['llvm_projects']: self.cfg['llvm_projects'] = [] if not self.cfg['llvm_runtimes']: diff --git a/easybuild/easyblocks/c/cplex.py b/easybuild/easyblocks/c/cplex.py index 9c4a894a4e..94c0bf2c20 100644 --- a/easybuild/easyblocks/c/cplex.py +++ b/easybuild/easyblocks/c/cplex.py @@ -60,6 +60,9 @@ def __init__(self, *args, **kwargs): self.with_python = False self.multi_python = 'Python' in self.cfg['multi_deps'] + # Bypass the .mod file check for GCCcore installs + self.cfg['skip_mod_files_sanity_check'] = True + def prepare_step(self, *args, **kwargs): """Prepare build environment.""" super(EB_CPLEX, self).prepare_step(*args, **kwargs) From dca450cd5102d8199ca0ba9938ec361269b01fd4 Mon Sep 17 00:00:00 2001 From: jfgrimm Date: Fri, 5 Jan 2024 13:46:00 +0000 Subject: [PATCH 142/145] set shortver in init --- easybuild/easyblocks/d/dualsphysics.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/easybuild/easyblocks/d/dualsphysics.py b/easybuild/easyblocks/d/dualsphysics.py index cde522aa80..9e8d792b16 100644 --- a/easybuild/easyblocks/d/dualsphysics.py +++ b/easybuild/easyblocks/d/dualsphysics.py @@ -59,7 +59,7 @@ def __init__(self, *args, **kwargs): super(EB_DualSPHysics, self).__init__(*args, **kwargs) self.dsph_target = None - self.shortver = None + self.shortver = '.'.join(self.version.split('.')[0:2]) def prepare_step(self, *args, **kwargs): """Determine name of binary that will be installed.""" @@ -70,8 +70,6 @@ def prepare_step(self, *args, **kwargs): else: self.dsph_target = 'CPU' - self.shortver = '.'.join(self.version.split('.')[0:2]) - def configure_step(self): """Custom configure procedure for DualSPHysics.""" srcdir = os.path.join(self.cfg['start_dir'], 'src/source') @@ -134,8 +132,6 @@ def sanity_check_step(self): else: self.dsph_target = 'CPU' - self.shortver = '.'.join(self.version.split('.')[0:2]) - bins = ['GenCase', 'PartVTK', 'IsoSurface', 'MeasureTool', 'GenCase_MkWord', 'DualSPHysics4.0_LiquidGas', 'DualSPHysics4.0_LiquidGasCPU', 'DualSPHysics%s' % self.shortver, 'DualSPHysics%s%s' % (self.shortver, self.dsph_target), 'DualSPHysics%s_NNewtonian' % self.shortver, From f7c3e019086c1d83f665b1dd137f2ca6b4e6bf6e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 5 Jan 2024 21:32:02 +0100 Subject: [PATCH 143/145] fix link order in PETSc easyblock for SCOTCH >= 7.x --- easybuild/easyblocks/p/petsc.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/easybuild/easyblocks/p/petsc.py b/easybuild/easyblocks/p/petsc.py index d5081d528a..1820d565d0 100644 --- a/easybuild/easyblocks/p/petsc.py +++ b/easybuild/easyblocks/p/petsc.py @@ -237,10 +237,10 @@ def configure_step(self): inc_spec = "-include=[%s]" % ','.join(scotch_inc) # For some reason there is a v3 suffix added to libptscotchparmetis - # which is the reason for this new code. - req_scotch_libs = ['libesmumps.a', 'libptesmumps.a', 'libptscotch.a', - 'libptscotcherr.a', 'libptscotchparmetisv3.a', 'libscotch.a', - 'libscotcherr.a'] + # which is the reason for this new code; + # note: order matters here, don't sort these alphabetically! + req_scotch_libs = ['libptesmumps.a', 'libptscotchparmetisv3.a', 'libptscotch.a', + 'libptscotcherr.a', 'libesmumps.a', 'libscotch.a', 'libscotcherr.a'] scotch_libs = [os.path.join(scotch, "lib", x) for x in req_scotch_libs] lib_spec = "-lib=[%s]" % ','.join(scotch_libs) self.cfg.update('configopts', ' '.join([withdep + spec for spec in ['=1', inc_spec, lib_spec]])) From 85625a32b007d34e26a22335175cc29443b94078 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Sun, 7 Jan 2024 17:37:29 +0100 Subject: [PATCH 144/145] Update description --- easybuild/easyblocks/generic/pythonpackage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 08a1b49632..b22c11a6b0 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -345,7 +345,7 @@ def extra_options(extra_vars=None): 'pip_no_index': [None, "Pass --no-index to pip to disable connecting to PyPi entirely which also disables " "the pip version check. Enabled by default when pip_ignore_installed=True", CUSTOM], 'pip_verbose': [None, "Pass --verbose to 'pip install' (if pip is used). " - "Defaults to 'True' if the EB option --debug is used.", CUSTOM], + "Enabled by default if the EB option --debug is used.", CUSTOM], 'req_py_majver': [None, "Required major Python version (only relevant when using system Python)", CUSTOM], 'req_py_minver': [None, "Required minor Python version (only relevant when using system Python)", CUSTOM], 'sanity_pip_check': [False, "Run 'python -m pip check' to ensure all required Python packages are " From 40d11520d200be2a2ccbc83e91e3f10f0149fa81 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 15 Jan 2024 11:54:12 +0100 Subject: [PATCH 145/145] use 5.0.x branch of framework when running easyblocks test suite outside of PR context --- .github/workflows/unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 75182c589d..8b6ac1804b 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -84,7 +84,7 @@ jobs: - name: install EasyBuild framework run: | # first determine which branch of easybuild-framework repo to install - BRANCH=develop + BRANCH=5.0.x if [ "x$GITHUB_BASE_REF" = 'xmain' ]; then BRANCH=main; fi if [ "x$GITHUB_BASE_REF" = 'x5.0.x' ]; then BRANCH=5.0.x; fi echo "Using easybuild-framework branch $BRANCH (\$GITHUB_BASE_REF $GITHUB_BASE_REF)"