Merge pull request #299 from caracal-pipeline/SpheMakh-patch-1

Sphe makh patch 1

.github/workflows/python-package.yml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
 
     steps:
     - uses: actions/checkout@v3

scabha/basetypes.py

@@ -145,6 +145,9 @@ def get_filelikes(dtype, value, filelikes=None):
 
     filelikes = set() if filelikes is None else filelikes
 
+    if value is UNSET or type(value) is UNSET:
+      return []
+
     origin = get_origin(dtype)
     args = get_args(dtype)
 

scabha/configuratt/deps.py

@@ -18,6 +18,7 @@ class FailRecord(object):
     origin: Optional[str] = None
     modulename: Optional[str] = None
     fname: Optional[str] = None
+    warn: bool = True
 
 @dataclass
 class RequirementRecord(object):

scabha/configuratt/resolvers.py

@@ -222,8 +222,11 @@ def load_include_files(keyword):
                         if match:
                             incl = match.group(1)
                             flags = set([x.strip().lower() for x in match.group(2).split(",")])
+                            warn = 'warn' in flags
+                            optional = 'optional' in flags
                         else:
                             flags = {}
+                            warn = optional = False
 
                         # check for (location)filename.yaml or (location)/filename.yaml style
                         match = re.match("^\\((.+)\\)/?(.+)$", incl)
@@ -232,37 +235,52 @@ def load_include_files(keyword):
                             if modulename.startswith("."):
                                 filename = os.path.join(os.path.dirname(pathname), modulename, filename)
                                 if not os.path.exists(filename):
-                                    if 'optional' in flags:
-                                        dependencies.add_fail(FailRecord(filename, pathname))
-                                        if 'warn' in flags:
+                                    if optional:
+                                        dependencies.add_fail(FailRecord(filename, pathname,warn=warn))
+                                        if warn:
                                             print(f"Warning: unable to find optional include {incl} ({filename})")
                                         continue
                                     raise ConfigurattError(f"{errloc}: {keyword} {incl} does not exist")
                             else:
                                 try:
                                     mod = importlib.import_module(modulename)
                                 except ImportError as exc:
-                                    if 'optional' in flags:
-                                        dependencies.add_fail(FailRecord(incl, pathname, modulename=modulename, fname=filename))
-                                        if 'warn' in flags:
+                                    if optional:
+                                        dependencies.add_fail(FailRecord(incl, pathname, modulename=modulename, 
+                                                                         fname=filename, warn=warn))
+                                        if warn:
                                             print(f"Warning: unable to import module for optional include {incl}")
                                         continue
                                     raise ConfigurattError(f"{errloc}: {keyword} {incl}: can't import {modulename} ({exc})")
-
-                                filename = os.path.join(os.path.dirname(mod.__file__), filename)
+                                if mod.__file__ is not None:
+                                    path = os.path.dirname(mod.__file__)
+                                else:
+                                    path = getattr(mod, '__path__', None)
+                                    if path is None:
+                                        if optional:
+                                            dependencies.add_fail(FailRecord(incl, pathname, modulename=modulename, 
+                                                                             fname=filename, warn=warn))
+                                            if warn:
+                                                print(f"Warning: unable to resolve path for optional include {incl}, does {modulename} contain __init__.py?")
+                                            continue
+                                        raise ConfigurattError(f"{errloc}: {keyword} {incl}: can't resolve path for {modulename}, does it contain __init__.py?")
+                                    path = path[0]
+
+                                filename = os.path.join(path, filename)
                                 if not os.path.exists(filename):
-                                    if 'optional' in flags:
-                                        dependencies.add_fail(FailRecord(incl, pathname, modulename=modulename, fname=filename))
-                                        if 'warn' in flags:
+                                    if optional:
+                                        dependencies.add_fail(FailRecord(incl, pathname, modulename=modulename, 
+                                                                         fname=filename, warn=warn))
+                                        if warn:
                                             print(f"Warning: unable to find optional include {incl}")
                                         continue
                                     raise ConfigurattError(f"{errloc}: {keyword} {incl}: {filename} does not exist")
                         # absolute path -- one candidate
                         elif os.path.isabs(incl):
                             if not os.path.exists(incl):
-                                if 'optional' in flags:
-                                    dependencies.add_fail(FailRecord(incl, pathname))
-                                    if 'warn' in flags:
+                                if optional:
+                                    dependencies.add_fail(FailRecord(incl, pathname, warn=warn))
+                                    if warn:
                                         print(f"Warning: unable to find optional include {incl}")
                                     continue
                                 raise ConfigurattError(f"{errloc}: {keyword} {incl} does not exist")
@@ -275,9 +293,9 @@ def load_include_files(keyword):
                                 if os.path.exists(filename):
                                     break
                             else:
-                                if 'optional' in flags:
-                                    dependencies.add_fail(FailRecord(incl, pathname))
-                                    if 'warn' in flags:
+                                if optional:
+                                    dependencies.add_fail(FailRecord(incl, pathname, warn=warn))
+                                    if warn:
                                         print(f"Warning: unable to find optional include {incl}")
                                     continue
                                 raise ConfigurattError(f"{errloc}: {keyword} {incl} not found in {':'.join(paths)}")

stimela/backends/flavours/python_flavours.py

@@ -51,13 +51,15 @@ def get_python_interpreter_args(cab: Cab, subst: Dict[str, Any], virtual_env: Op
     # get virtual env, if specified
     if virtual_env:
         virtual_env = os.path.expanduser(virtual_env)
-        interpreter = f"{virtual_env}/bin/python"
+        interpreter = f"{virtual_env}/bin/{cab.flavour.interpreter_binary}"
         if not os.path.isfile(interpreter):
-            raise CabValidationError(f"virtual environment {virtual_env} doesn't exist")
+            raise CabValidationError(f"{interpreter} doesn't exist")
     else:
-        interpreter = "python"
+        interpreter = cab.flavour.interpreter_binary
 
-    return [interpreter, "-u"]
+    args =  cab.flavour.interpreter_command.format(python=interpreter).split()
+
+    return args
 
 
 @dataclass
@@ -67,6 +69,10 @@ class PythonCallableFlavour(_CallableFlavour):
     expected to be in the form of [package.]module.function
     """
     kind: str = "python"
+    # name of python binary to use  
+    interpreter_binary: str = "python"
+    # Full command used to launch interpreter. {python} gets substituted for the interpreter path
+    interpreter_command: str = "{python} -u"
     # don't log full command by default, as that's full of code
     log_full_command: bool = False
 
@@ -155,6 +161,10 @@ class PythonCodeFlavour(_BaseFlavour):
     output_vars: bool = True
     # if True, command will have {}-substitutions done on it
     subst: bool = False
+    # name of python binary to use  
+    interpreter_binary: str = "python"
+    # Full command used to launch interpreter. {python} gets substituted for the interpreter path
+    interpreter_command: str = "{python} -u"
     # don't log full command by default, as that's full of code
     log_full_command: bool = False
 

stimela/commands/run.py

@@ -107,9 +107,10 @@ def load_recipe_files(filenames: List[str]):
         full_deps.update(deps)
 
     # warn user if any includes failed
-    if full_deps.fails:
-        logger().warning(f"{len(full_deps.fails)} optional includes were not found, some cabs may not be available")
-        for path, dep in full_deps.fails.items():
+    failed_includes = {path: dep for path, dep in full_deps.fails.items() if dep.warn}
+    if failed_includes:
+        logger().warning(f"{len(failed_includes)} optional includes were not found")
+        for path, dep in failed_includes.items():
             logger().warning(f"    {path} (from {dep.origin})")
 
     # merge into full config dependencies

stimela/config.py

@@ -5,6 +5,7 @@
 from omegaconf.omegaconf import OmegaConf
 from omegaconf.errors import OmegaConfBaseException
 from collections import OrderedDict
+import psutil
 
 from yaml.error import YAMLError
 import stimela
@@ -268,9 +269,12 @@ def _load(conf, config_file):
     runtime = dict(
         date=_ds, 
         time=_ts, datetime=f"{_ds}-{_ts}", 
+        ncpu=psutil.cpu_count(logical=True),
         node=platform.node().split('.', 1)[0],
         hostname=platform.node(), 
         env={key: value.replace('${', '\${') for key, value in os.environ.items()})
+    runtime['ncpu-logical'] = psutil.cpu_count(logical=True)
+    runtime['ncpu-physical'] = psutil.cpu_count(logical=False)
 
     conf.run = OmegaConf.create(runtime)