Merge pull request #136 from PanDAWMS/next

3.7.8.21

PILOTVERSION

@@ -1 +1 @@
-3.7.7.3
+3.7.8.21

pilot.py

@@ -31,6 +31,7 @@
 import time
 from os import getcwd, chdir, environ
 from os.path import exists, join
+from re import match
 from shutil import rmtree
 from typing import Any
 
@@ -243,6 +244,25 @@ def str2bool(var: str) -> bool:
     return ret
 
 
+def validate_resource_type(value: str) -> str:
+    """
+    Validate the resource type.
+
+    :param value: resource type (str)
+    :return: resource type (str)
+    :raises: argparse.ArgumentTypeError if the resource type is invalid.
+    """
+    # Define the allowed patterns
+    allowed_patterns = ["SCORE", "MCORE", "SCORE_*", "MCORE_*"]
+    if value in allowed_patterns:
+        return value
+    # Check for pattern matching
+    for pattern in allowed_patterns:
+        if pattern.endswith('*') and match(f"^{pattern[:-1]}", value):
+            return value
+    raise argparse.ArgumentTypeError(f"Invalid resource type: {value}")
+
+
 def get_args() -> Any:
     """
     Return the args from the arg parser.
@@ -548,9 +568,8 @@ def get_args() -> Any:
         "--resource-type",
         dest="resource_type",
         default="",
-        type=str,
-        choices=["SCORE", "MCORE", "SCORE_HIMEM", "MCORE_HIMEM"],
-        help="Resource type; MCORE, SCORE, SCORE_HIMEM or MCORE_HIMEM",
+        type=validate_resource_type,
+        help="Resource type; MCORE, SCORE or patterns like SCORE_* and MCORE_*",
     )
     arg_parser.add_argument(
         "--use-https",

pilot/control/job.py

@@ -788,9 +788,14 @@ def get_data_structure(job: Any, state: str, args: Any, xml: str = "", metadata:
             extra = {'readbyterate': readfrac}
     else:
         logger.debug('read_bytes info not yet available')
+
     # extract and remove any GPU info from data since it will be reported with job metrics
     add_gpu_info(data, extra)
 
+    # add the lsetup time if set
+    if job.lsetuptime:
+        extra['lsetup_time'] = job.lsetuptime
+
     job_metrics = get_job_metrics(job, extra=extra)
     if job_metrics:
         data['jobMetrics'] = job_metrics

pilot/eventservice/esprocess/esprocess.py

@@ -163,7 +163,7 @@ def init_yampl_socket(self, executable: str) -> str:
         is_mt = "--multithreaded=true" in executable.lower()
         if is_ca:
             if is_mt:
-                preexec_socket_config = f" --mtes=True --mtes-channel=\"{socket_name}\" "
+                preexec_socket_config = f" --mtes --mtes-channel=\"{socket_name}\" "
             else:
                 preexec_socket_config = f" --preExec 'ConfigFlags.MP.EventRangeChannel=\"{socket_name}\"' "
         else: