From 61d0059e09c0f817bb6ed0fae80368764bf1b9af Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 29 Mar 2020 13:03:02 +0200 Subject: [PATCH 1/5] port to Python 3 --- bin/mympisanity.py | 2 +- lib/vsc/mympirun/common.py | 14 ++++++++++++-- lib/vsc/mympirun/mpi/mpi.py | 2 +- lib/vsc/mympirun/pmi/slurm.py | 2 +- lib/vsc/mympirun/rm/pbs.py | 1 - setup.py | 6 +++--- test/end2end.py | 4 ++-- test/mpi.py | 11 +++++------ test/pmi_utils.py | 5 +++-- test/sched.py | 16 ++++++++++------ tox.ini | 4 ---- vsc-ci.ini | 1 + 12 files changed, 39 insertions(+), 29 deletions(-) diff --git a/bin/mympisanity.py b/bin/mympisanity.py index 1c16033b..073844aa 100644 --- a/bin/mympisanity.py +++ b/bin/mympisanity.py @@ -122,7 +122,7 @@ def check(): log.error("OMP_NUM_THREADS set for rank %s to %s does not match affinity width %s" % (rank, omp, af)) # check for mapping - for idx, x in enumerate(recvbuf): + for idx, _ in enumerate(recvbuf): next_idx = (idx + 1) % len(recvbuf) if recvbuf[idx]['hostname'] == recvbuf[next_idx]['hostname']: if not recvbuf[idx]['affinity'][-1] == recvbuf[next_idx]['affinity'][0] - 1: diff --git a/lib/vsc/mympirun/common.py b/lib/vsc/mympirun/common.py index a6810cf5..8a335f29 100644 --- a/lib/vsc/mympirun/common.py +++ b/lib/vsc/mympirun/common.py @@ -96,8 +96,18 @@ def sched_to_key(klass): # next, try to use the scheduler defined by environment variables for sched in found_sched: - nodeinfo = not hasattr(sched, 'SCHED_ENVIRON_NODE_INFO') or sched.SCHED_ENVIRON_NODE_INFO in os.environ - if nodeinfo and sched.SCHED_ENVIRON_ID in os.environ: + + # take into account that SCHED_ENVIRON_NODE_INFO can be None, + # and chechking "None in os.environ" fails hard in Python 3 (string value is required) + if hasattr(sched, 'SCHED_ENVIRON_NODE_INFO'): + if sched.SCHED_ENVIRON_NODE_INFO is not None: + nodeinfo = sched.SCHED_ENVIRON_NODE_INFO in os.environ + else: + nodeinfo = False + else: + nodeinfo = True + + if nodeinfo and sched.SCHED_ENVIRON_ID is not None and sched.SCHED_ENVIRON_ID in os.environ: return sched, found_sched # If that fails, try to force the local scheduler diff --git a/lib/vsc/mympirun/mpi/mpi.py b/lib/vsc/mympirun/mpi/mpi.py index a87de45b..5dc48536 100644 --- a/lib/vsc/mympirun/mpi/mpi.py +++ b/lib/vsc/mympirun/mpi/mpi.py @@ -662,7 +662,7 @@ def set_mpiexec_opts_from_env(self): """ # get all unique variables that are both in os.environ and in OPTS_FROM_ENV_BASE - vars_to_pass = nub(filter(os.environ.has_key, self.OPTS_FROM_ENV_BASE)) + vars_to_pass = nub(filter(lambda key: key in os.environ, self.OPTS_FROM_ENV_BASE)) self.mpiexec_opts_from_env.extend(vars_to_pass) prefixes = self.OPTS_FROM_ENV_FLAVOR_PREFIX + self.OPTS_FROM_ENV_BASE_PREFIX + self.options.variablesprefix diff --git a/lib/vsc/mympirun/pmi/slurm.py b/lib/vsc/mympirun/pmi/slurm.py index 61835d05..814ea991 100644 --- a/lib/vsc/mympirun/pmi/slurm.py +++ b/lib/vsc/mympirun/pmi/slurm.py @@ -147,7 +147,7 @@ def job_info(self, job_info): if 'SLURM_JOB_GPUS' in os.environ: try: # this should fail when slurm switched to compressed repr (eg 0-3 instead of current 0,1,2,3) - ngpus = len(map(int, os.environ['SLURM_JOB_GPUS'].split(','))) + ngpus = len(list(map(int, os.environ['SLURM_JOB_GPUS'].split(',')))) except Exception as e: self.log.raiseException("Failed to get the number of gpus per node from %s: %s" % (dbgtxt, e)) diff --git a/lib/vsc/mympirun/rm/pbs.py b/lib/vsc/mympirun/rm/pbs.py index 1ad65a34..a8a841c8 100644 --- a/lib/vsc/mympirun/rm/pbs.py +++ b/lib/vsc/mympirun/rm/pbs.py @@ -56,7 +56,6 @@ def __init__(self, *args, **kwargs): if which(PBSSSH) and which(PBSDSH): self.log.debug("Both 'pbsssh' and 'pbsdsh' found, so using 'pbsssh' as remote shell command.") self.RSH_LARGE_CMD = PBSSSH - self.RSH_LARGE_LIMIT = PBSSSH self.HYDRA_LAUNCHER_EXEC = PBSSSH elif which(PBSSSH): self.log.debug("Can't use '%s' wrapper if '%s' is not available", PBSDSH, PBSSSH) diff --git a/setup.py b/setup.py index da6ceceb..690bcd2d 100644 --- a/setup.py +++ b/setup.py @@ -40,14 +40,14 @@ PACKAGE = { 'install_requires': [ - 'vsc-base >= 2.9.3', - 'vsc-install >= 0.10.25', # for modified subclassing + 'vsc-base >= 3.0.1', + 'vsc-install >= 0.15.1', 'IPy', ], 'tests_require': [ 'mock', ], - 'version': '5.0.1', + 'version': '5.1.0', 'author': [sdw, kh], 'maintainer': [sdw, kh], 'zip_safe': False, diff --git a/test/end2end.py b/test/end2end.py index 6ef1fecc..bb47d195 100644 --- a/test/end2end.py +++ b/test/end2end.py @@ -116,7 +116,7 @@ def setUp(self): os.environ['PATH'] = '%s:%s' % (os.path.join(self.tmpdir, 'bin'), os.getenv('PATH', '')) # make sure we're using the right mympirun installation... - ec, out = run([sys.executable, '-c', "import vsc.mympirun; print vsc.mympirun.__file__"]) + ec, out = run([sys.executable, '-c', "import vsc.mympirun; print(vsc.mympirun.__file__)"]) out = out.strip() expected_path = os.path.join(self.topdir, 'lib', 'vsc', 'mympirun') self.assertTrue(os.path.samefile(os.path.dirname(out), expected_path), "%s not in %s" % (out, expected_path)) @@ -341,7 +341,7 @@ def test_env_variables(self): ] ec, out = run(command) - for key in nub(filter(os.environ.has_key, MPI.OPTS_FROM_ENV_BASE)): + for key in nub(filter(lambda key: key in os.environ, MPI.OPTS_FROM_ENV_BASE)): self.assertTrue(key in out, "%s is not in out" % key) regex = r'.*-envlist [^ ]*USER.*' diff --git a/test/mpi.py b/test/mpi.py index dcbe01b8..1d865da2 100644 --- a/test/mpi.py +++ b/test/mpi.py @@ -33,7 +33,6 @@ import pkgutil import re import stat -import string from vsc.install.testing import TestCase from vsc.utils.run import run @@ -112,9 +111,9 @@ def test_which(self): raise Exception("Something went wrong while trying to run `which`: %s" % unixwhich) self.assertTrue(mpiwhich, msg="mpi which did not return anything, (unix which: %s" % unixwhich) - self.assertEqual(mpiwhich, string.strip(unixwhich), - msg="the return values of unix which and which() aren't"" the same: %s != %s" % - (mpiwhich, string.strip(unixwhich))) + self.assertEqual(mpiwhich, unixwhich.strip(), + msg="the return values of unix which and which() aren't the same: %s != %s" % + (mpiwhich, unixwhich.strip())) ################### ## MPI functions ## @@ -201,7 +200,7 @@ def test_set_netmask(self): # matches "IP address / netmask" reg = re.compile(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}") print("netmask: %s" % mpi_instance.netmask) - for substr in string.split(mpi_instance.netmask, sep=":"): + for substr in mpi_instance.netmask.split(':'): try: IP(substr) except ValueError: @@ -312,7 +311,7 @@ def test_set_mpiexec_opts_from_env(self): """test if mpiexec_opts_from_env only contains environment variables that start with the given prefix""" mpi_instance = getinstance(mpim.MPI, Local, MympirunOption()) - if not os.environ.has_key('PYTHONPATH'): + if 'PYTHONPATH' not in os.environ: os.environ[key] = "/example:/test/123" mpi_instance.set_mpiexec_opts_from_env() prefixes = mpi_instance.OPTS_FROM_ENV_FLAVOR_PREFIX diff --git a/test/pmi_utils.py b/test/pmi_utils.py index 24466331..542103ce 100644 --- a/test/pmi_utils.py +++ b/test/pmi_utils.py @@ -32,6 +32,7 @@ import re from vsc.install.testing import TestCase +from vsc.utils.py2vs3 import is_string from vsc.utils.run import run from sched import reset_env @@ -78,7 +79,7 @@ def setUp(self): os.environ['PYTHONPATH'] = '%s:%s:%s' % (eggs, lib, os.getenv('PYTHONPATH', '')) # make sure we're using the right mympirun installation... - ec, out = run([sys.executable, '-c', "import vsc.mympirun; print vsc.mympirun.__file__"]) + ec, out = run([sys.executable, '-c', "import vsc.mympirun; print(vsc.mympirun.__file__)"]) out = out.strip() expected_path = os.path.join(self.topdir, 'lib', 'vsc', 'mympirun') self.assertTrue(os.path.samefile(os.path.dirname(out), expected_path), "%s not in %s" % (out, expected_path)) @@ -118,7 +119,7 @@ def set_mpi(self, name, version): return mpirun def set_env(self, env): - if isinstance(env, basestring): + if is_string(env): for line in env.split("\n"): if '=' in line: os.environ.update(dict([line.strip().split("=", 1)])) diff --git a/test/sched.py b/test/sched.py index daf601b7..8aaeef4b 100644 --- a/test/sched.py +++ b/test/sched.py @@ -104,7 +104,7 @@ def set_SLURM_env(tmpdir): def reset_env(orig_env): """Reset environment to provided original environment.""" - for key in nub(os.environ.keys() + orig_env.keys()): + for key in nub(list(os.environ.keys()) + list(orig_env.keys())): orig_val = orig_env.get(key) if orig_val is None: if key in os.environ: @@ -135,7 +135,7 @@ def test_what_sched(self): expected_found_sched = [SLURM, Local, PBS, Scoop] # if scheduler is specified, then just return corresponding class - for key, val in SCHEDDICT.iteritems(): + for key, val in SCHEDDICT.items(): sched, found_sched = what_sched(key, schedm) self.assertEqual(sched, val) self.assertEqual(found_sched, expected_found_sched) @@ -179,14 +179,18 @@ def test_get_id(self): get_id gets called by the __init__ of getinstance() """ - for key, val in SCHEDDICT.iteritems(): + for key, val in SCHEDDICT.items(): if key == 'pbs': set_PBS_env(self.tmpdir) elif key == 'slurm': set_SLURM_env(self.tmpdir) inst = getinstance(mpim.MPI, val, MympirunOption()) - self.assertTrue(inst.sched_id == os.environ.get(inst.SCHED_ENVIRON_ID, None) or + expected = None + if inst.SCHED_ENVIRON_ID is not None: + expected = os.environ.get(inst.SCHED_ENVIRON_ID) + + self.assertTrue(inst.sched_id == expected or inst.sched_id.startswith("SCHED_%s" % inst.__class__.__name__)) def test_core_on_this_node(self): @@ -195,7 +199,7 @@ def test_core_on_this_node(self): core_on_this_node() gets called by the __init__ of getinstance() """ - for key, val in SCHEDDICT.iteritems(): + for key, val in SCHEDDICT.items(): if key == 'pbs': set_PBS_env(self.tmpdir) elif key == 'slurm': @@ -210,7 +214,7 @@ def test_which_cpus(self): which_cpus() gets called by the __init__ of getinstance() """ - for key, val in SCHEDDICT.iteritems(): + for key, val in SCHEDDICT.items(): if key == 'pbs': set_PBS_env(self.tmpdir) elif key == 'slurm': diff --git a/tox.ini b/tox.ini index 5ac0512f..a26abd24 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,6 @@ [tox] envlist = py27,py36 skipsdist = true -skip_missing_interpreters = true [testenv] commands_pre = @@ -13,6 +12,3 @@ commands_pre = python -m easy_install -U vsc-install commands = python setup.py test passenv = USER - -[testenv:py36] -ignore_outcome = true diff --git a/vsc-ci.ini b/vsc-ci.ini index 7d4ee881..3ef48149 100644 --- a/vsc-ci.ini +++ b/vsc-ci.ini @@ -1,2 +1,3 @@ [vsc-ci] pip3_install_tox=1 +py3_tests_must_pass=1 From 14c2d00fa35a591ac625a8b2da93c255df6f867f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 29 Apr 2020 15:53:28 +0200 Subject: [PATCH 2/5] delete .eggs directory before running tests --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index a26abd24..466d0d64 100644 --- a/tox.ini +++ b/tox.ini @@ -10,5 +10,6 @@ skipsdist = true commands_pre = pip install 'setuptools<42.0' python -m easy_install -U vsc-install + rm -rf .eggs commands = python setup.py test passenv = USER From 0b8ed147dc0cdbab214fe19ec5301ec2182156c3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 30 Jun 2020 10:22:00 +0200 Subject: [PATCH 3/5] add .eggs* to .gitignore --- .gitignore | 2 +- tox.ini | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 7471fe1c..83fd769b 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,7 @@ dist setup.cfg build eggs -.eggs +.eggs* parts var sdist diff --git a/tox.ini b/tox.ini index 466d0d64..a26abd24 100644 --- a/tox.ini +++ b/tox.ini @@ -10,6 +10,5 @@ skipsdist = true commands_pre = pip install 'setuptools<42.0' python -m easy_install -U vsc-install - rm -rf .eggs commands = python setup.py test passenv = USER From 0032ddbc0ca0874f54dac7c1c11b70081556a110 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 30 Jun 2020 13:52:28 +0200 Subject: [PATCH 4/5] take into account that SCHED_ENVIRON_ID my not be a known attribute + fix typo in comment --- lib/vsc/mympirun/common.py | 17 +++++++---------- lib/vsc/mympirun/rm/sched.py | 1 + 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/lib/vsc/mympirun/common.py b/lib/vsc/mympirun/common.py index 8a335f29..9fac9bc1 100644 --- a/lib/vsc/mympirun/common.py +++ b/lib/vsc/mympirun/common.py @@ -97,18 +97,15 @@ def sched_to_key(klass): # next, try to use the scheduler defined by environment variables for sched in found_sched: + # determine whether environment variable for node info (like $PBS_NODEFILE, $SLURM_NODELIST) is defined; # take into account that SCHED_ENVIRON_NODE_INFO can be None, - # and chechking "None in os.environ" fails hard in Python 3 (string value is required) - if hasattr(sched, 'SCHED_ENVIRON_NODE_INFO'): - if sched.SCHED_ENVIRON_NODE_INFO is not None: - nodeinfo = sched.SCHED_ENVIRON_NODE_INFO in os.environ - else: - nodeinfo = False - else: - nodeinfo = True + # and checking "None in os.environ" fails hard in Python 3 (string value is required) + nodeinfo = getattr(sched, 'SCHED_ENVIRON_NODE_INFO', True) and (sched.SCHED_ENVIRON_NODE_INFO or '') in os.environ - if nodeinfo and sched.SCHED_ENVIRON_ID is not None and sched.SCHED_ENVIRON_ID in os.environ: - return sched, found_sched + if nodeinfo: + # determine whether environment variable that specifies job ID (like $PBS_JOBID, $SLURM_JOBID) is defined + if (getattr(sched, 'SCHED_ENVIRON_ID', None) or '') in os.environ: + return sched, found_sched # If that fails, try to force the local scheduler LOGGER.debug("No scheduler found in environment, trying local") diff --git a/lib/vsc/mympirun/rm/sched.py b/lib/vsc/mympirun/rm/sched.py index 2a626688..23a73714 100644 --- a/lib/vsc/mympirun/rm/sched.py +++ b/lib/vsc/mympirun/rm/sched.py @@ -42,6 +42,7 @@ class Sched(SchedBase): # if the SCHED_ENVIRON_ID is not found, create one yourself AUTOGENERATE_JOBID = False + SCHED_ENVIRON_ID = None SCHED_ENVIRON_NODE_INFO = None SAFE_RSH_CMD = 'ssh' From ead9a581a877b7d7d1cb8050325a63c0973f26ad Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 30 Jun 2020 14:04:50 +0200 Subject: [PATCH 5/5] correctly deal with SCHED_ENVIRON_NODE_INFO --- lib/vsc/mympirun/common.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/vsc/mympirun/common.py b/lib/vsc/mympirun/common.py index 9fac9bc1..d466e764 100644 --- a/lib/vsc/mympirun/common.py +++ b/lib/vsc/mympirun/common.py @@ -98,9 +98,13 @@ def sched_to_key(klass): for sched in found_sched: # determine whether environment variable for node info (like $PBS_NODEFILE, $SLURM_NODELIST) is defined; - # take into account that SCHED_ENVIRON_NODE_INFO can be None, - # and checking "None in os.environ" fails hard in Python 3 (string value is required) - nodeinfo = getattr(sched, 'SCHED_ENVIRON_NODE_INFO', True) and (sched.SCHED_ENVIRON_NODE_INFO or '') in os.environ + if hasattr(sched, 'SCHED_ENVIRON_NODE_INFO'): + # take into account that SCHED_ENVIRON_NODE_INFO can be None, + # and checking "None in os.environ" fails hard in Python 3 (string value is required) + nodeinfo = (sched.SCHED_ENVIRON_NODE_INFO or '') in os.environ + else: + # if SCHED_ENVIRON_NODE_INFO attribute does not exist, we still check SCHED_ENVIRON_ID below + nodeinfo = True if nodeinfo: # determine whether environment variable that specifies job ID (like $PBS_JOBID, $SLURM_JOBID) is defined