Merge pull request #166 from boegel/py3

port to Python 3

.gitignore

@@ -10,7 +10,7 @@ dist
 setup.cfg
 build
 eggs
-.eggs
+.eggs*
 parts
 var
 sdist

bin/mympisanity.py

@@ -122,7 +122,7 @@ def check():
                 log.error("OMP_NUM_THREADS set for rank %s to %s does not match affinity width %s" % (rank, omp, af))
 
     # check for mapping
-    for idx, x in enumerate(recvbuf):
+    for idx, _ in enumerate(recvbuf):
         next_idx = (idx + 1) % len(recvbuf)
         if recvbuf[idx]['hostname'] == recvbuf[next_idx]['hostname']:
             if not recvbuf[idx]['affinity'][-1] == recvbuf[next_idx]['affinity'][0] - 1:

lib/vsc/mympirun/common.py

@@ -96,9 +96,20 @@ def sched_to_key(klass):
 
     # next, try to use the scheduler defined by environment variables
     for sched in found_sched:
-        nodeinfo = not hasattr(sched, 'SCHED_ENVIRON_NODE_INFO') or sched.SCHED_ENVIRON_NODE_INFO in os.environ
-        if nodeinfo and sched.SCHED_ENVIRON_ID in os.environ:
-            return sched, found_sched
+
+        # determine whether environment variable for node info (like $PBS_NODEFILE, $SLURM_NODELIST) is defined;
+        if hasattr(sched, 'SCHED_ENVIRON_NODE_INFO'):
+            # take into account that SCHED_ENVIRON_NODE_INFO can be None,
+            # and checking "None in os.environ" fails hard in Python 3 (string value is required)
+            nodeinfo = (sched.SCHED_ENVIRON_NODE_INFO or '') in os.environ
+        else:
+            # if SCHED_ENVIRON_NODE_INFO attribute does not exist, we still check SCHED_ENVIRON_ID below
+            nodeinfo = True
+
+        if nodeinfo:
+            # determine whether environment variable that specifies job ID (like $PBS_JOBID, $SLURM_JOBID) is defined
+            if (getattr(sched, 'SCHED_ENVIRON_ID', None) or '') in os.environ:
+                return sched, found_sched
 
     # If that fails, try to force the local scheduler
     LOGGER.debug("No scheduler found in environment, trying local")

lib/vsc/mympirun/mpi/mpi.py

@@ -662,7 +662,7 @@ def set_mpiexec_opts_from_env(self):
         """
 
         # get all unique variables that are both in os.environ and in OPTS_FROM_ENV_BASE
-        vars_to_pass = nub(filter(os.environ.has_key, self.OPTS_FROM_ENV_BASE))
+        vars_to_pass = nub(filter(lambda key: key in os.environ, self.OPTS_FROM_ENV_BASE))
         self.mpiexec_opts_from_env.extend(vars_to_pass)
 
         prefixes = self.OPTS_FROM_ENV_FLAVOR_PREFIX + self.OPTS_FROM_ENV_BASE_PREFIX + self.options.variablesprefix

lib/vsc/mympirun/pmi/slurm.py

@@ -147,7 +147,7 @@ def job_info(self, job_info):
         if 'SLURM_JOB_GPUS' in os.environ:
             try:
                 # this should fail when slurm switched to compressed repr (eg 0-3 instead of current 0,1,2,3)
-                ngpus = len(map(int, os.environ['SLURM_JOB_GPUS'].split(',')))
+                ngpus = len(list(map(int, os.environ['SLURM_JOB_GPUS'].split(','))))
             except Exception as e:
                 self.log.raiseException("Failed to get the number of gpus per node from %s: %s" % (dbgtxt, e))
 

lib/vsc/mympirun/rm/pbs.py

@@ -56,7 +56,6 @@ def __init__(self, *args, **kwargs):
         if which(PBSSSH) and which(PBSDSH):
             self.log.debug("Both 'pbsssh' and 'pbsdsh' found, so using 'pbsssh' as remote shell command.")
             self.RSH_LARGE_CMD = PBSSSH
-            self.RSH_LARGE_LIMIT = PBSSSH
             self.HYDRA_LAUNCHER_EXEC = PBSSSH
         elif which(PBSSSH):
             self.log.debug("Can't use '%s' wrapper if '%s' is not available", PBSDSH, PBSSSH)

lib/vsc/mympirun/rm/sched.py

@@ -42,6 +42,7 @@ class Sched(SchedBase):
     # if the SCHED_ENVIRON_ID is not found, create one yourself
     AUTOGENERATE_JOBID = False
 
+    SCHED_ENVIRON_ID = None
     SCHED_ENVIRON_NODE_INFO = None
 
     SAFE_RSH_CMD = 'ssh'

setup.py

@@ -40,14 +40,14 @@
 
 PACKAGE = {
     'install_requires': [
-        'vsc-base >= 2.9.3',
-        'vsc-install >= 0.10.25',  # for modified subclassing
+        'vsc-base >= 3.0.1',
+        'vsc-install >= 0.15.1',
         'IPy',
     ],
     'tests_require': [
         'mock',
     ],
-    'version': '5.0.1',
+    'version': '5.1.0',
     'author': [sdw, kh],
     'maintainer': [sdw, kh],
     'zip_safe': False,

test/end2end.py

@@ -116,7 +116,7 @@ def setUp(self):
         os.environ['PATH'] = '%s:%s' % (os.path.join(self.tmpdir, 'bin'), os.getenv('PATH', ''))
 
         # make sure we're using the right mympirun installation...
-        ec, out = run([sys.executable, '-c', "import vsc.mympirun; print vsc.mympirun.__file__"])
+        ec, out = run([sys.executable, '-c', "import vsc.mympirun; print(vsc.mympirun.__file__)"])
         out = out.strip()
         expected_path = os.path.join(self.topdir, 'lib', 'vsc', 'mympirun')
         self.assertTrue(os.path.samefile(os.path.dirname(out), expected_path), "%s not in %s" % (out, expected_path))
@@ -341,7 +341,7 @@ def test_env_variables(self):
         ]
         ec, out = run(command)
 
-        for key in nub(filter(os.environ.has_key, MPI.OPTS_FROM_ENV_BASE)):
+        for key in nub(filter(lambda key: key in os.environ, MPI.OPTS_FROM_ENV_BASE)):
             self.assertTrue(key in out, "%s is not in out" % key)
 
         regex = r'.*-envlist [^ ]*USER.*'

test/mpi.py

@@ -33,7 +33,6 @@
 import pkgutil
 import re
 import stat
-import string
 
 from vsc.install.testing import TestCase
 from vsc.utils.run import run
@@ -112,9 +111,9 @@ def test_which(self):
                 raise Exception("Something went wrong while trying to run `which`: %s" % unixwhich)
 
             self.assertTrue(mpiwhich, msg="mpi which did not return anything, (unix which: %s" % unixwhich)
-            self.assertEqual(mpiwhich, string.strip(unixwhich),
-                             msg="the return values of unix which and which() aren't"" the same: %s != %s" %
-                             (mpiwhich, string.strip(unixwhich)))
+            self.assertEqual(mpiwhich, unixwhich.strip(),
+                             msg="the return values of unix which and which() aren't the same: %s != %s" %
+                             (mpiwhich, unixwhich.strip()))
 
      ###################
      ## MPI functions ##
@@ -201,7 +200,7 @@ def test_set_netmask(self):
         # matches "IP address / netmask"
         reg = re.compile(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}")
         print("netmask: %s" % mpi_instance.netmask)
-        for substr in string.split(mpi_instance.netmask, sep=":"):
+        for substr in mpi_instance.netmask.split(':'):
             try:
                 IP(substr)
             except ValueError:
@@ -312,7 +311,7 @@ def test_set_mpiexec_opts_from_env(self):
         """test if mpiexec_opts_from_env only contains environment variables that start with the given prefix"""
         mpi_instance = getinstance(mpim.MPI, Local, MympirunOption())
 
-        if not os.environ.has_key('PYTHONPATH'):
+        if 'PYTHONPATH' not in os.environ:
             os.environ[key] = "/example:/test/123"
         mpi_instance.set_mpiexec_opts_from_env()
         prefixes = mpi_instance.OPTS_FROM_ENV_FLAVOR_PREFIX

test/pmi_utils.py

@@ -32,6 +32,7 @@
 import re
 
 from vsc.install.testing import TestCase
+from vsc.utils.py2vs3 import is_string
 from vsc.utils.run import run
 
 from sched import reset_env
@@ -78,7 +79,7 @@ def setUp(self):
         os.environ['PYTHONPATH'] = '%s:%s:%s' % (eggs, lib, os.getenv('PYTHONPATH', ''))
 
         # make sure we're using the right mympirun installation...
-        ec, out = run([sys.executable, '-c', "import vsc.mympirun; print vsc.mympirun.__file__"])
+        ec, out = run([sys.executable, '-c', "import vsc.mympirun; print(vsc.mympirun.__file__)"])
         out = out.strip()
         expected_path = os.path.join(self.topdir, 'lib', 'vsc', 'mympirun')
         self.assertTrue(os.path.samefile(os.path.dirname(out), expected_path), "%s not in %s" % (out, expected_path))
@@ -118,7 +119,7 @@ def set_mpi(self, name, version):
         return mpirun
 
     def set_env(self, env):
-        if isinstance(env, basestring):
+        if is_string(env):
             for line in env.split("\n"):
                 if '=' in line:
                     os.environ.update(dict([line.strip().split("=", 1)]))

test/sched.py

@@ -104,7 +104,7 @@ def set_SLURM_env(tmpdir):
 
 def reset_env(orig_env):
     """Reset environment to provided original environment."""
-    for key in nub(os.environ.keys() + orig_env.keys()):
+    for key in nub(list(os.environ.keys()) + list(orig_env.keys())):
         orig_val = orig_env.get(key)
         if orig_val is None:
             if key in os.environ:
@@ -135,7 +135,7 @@ def test_what_sched(self):
         expected_found_sched = [SLURM, Local, PBS, Scoop]
 
         # if scheduler is specified, then just return corresponding class
-        for key, val in SCHEDDICT.iteritems():
+        for key, val in SCHEDDICT.items():
             sched, found_sched = what_sched(key, schedm)
             self.assertEqual(sched, val)
             self.assertEqual(found_sched, expected_found_sched)
@@ -179,14 +179,18 @@ def test_get_id(self):
 
         get_id gets called by the __init__ of getinstance()
         """
-        for key, val in SCHEDDICT.iteritems():
+        for key, val in SCHEDDICT.items():
             if key == 'pbs':
                 set_PBS_env(self.tmpdir)
             elif key == 'slurm':
                 set_SLURM_env(self.tmpdir)
 
             inst = getinstance(mpim.MPI, val, MympirunOption())
-            self.assertTrue(inst.sched_id == os.environ.get(inst.SCHED_ENVIRON_ID, None) or
+            expected = None
+            if inst.SCHED_ENVIRON_ID is not None:
+                expected = os.environ.get(inst.SCHED_ENVIRON_ID)
+
+            self.assertTrue(inst.sched_id == expected or
                             inst.sched_id.startswith("SCHED_%s" % inst.__class__.__name__))
 
     def test_core_on_this_node(self):
@@ -195,7 +199,7 @@ def test_core_on_this_node(self):
 
         core_on_this_node() gets called by the __init__ of getinstance()
         """
-        for key, val in SCHEDDICT.iteritems():
+        for key, val in SCHEDDICT.items():
             if key == 'pbs':
                 set_PBS_env(self.tmpdir)
             elif key == 'slurm':
@@ -210,7 +214,7 @@ def test_which_cpus(self):
 
         which_cpus() gets called by the __init__ of getinstance()
         """
-        for key, val in SCHEDDICT.iteritems():
+        for key, val in SCHEDDICT.items():
             if key == 'pbs':
                 set_PBS_env(self.tmpdir)
             elif key == 'slurm':

tox.ini

@@ -5,14 +5,10 @@
 [tox]
 envlist = py27,py36
 skipsdist = true
-skip_missing_interpreters = true
 
 [testenv]
 commands_pre =
     pip install 'setuptools<42.0'
     python -m easy_install -U vsc-install
 commands = python setup.py test
 passenv = USER
-
-[testenv:py36]
-ignore_outcome = true

vsc-ci.ini

@@ -1,2 +1,3 @@
 [vsc-ci]
 pip3_install_tox=1
+py3_tests_must_pass=1