From df549fba3e83be66cee5bcac3249d0931f355de0 Mon Sep 17 00:00:00 2001 From: Alan Malta Rodrigues Date: Fri, 19 Jul 2024 08:20:15 -0400 Subject: [PATCH 1/2] Only dump CMSSW process attributes in the logs when needed format the large line with pformat --- src/python/WMCore/WMRuntime/Scripts/SetupCMSSWPset.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/python/WMCore/WMRuntime/Scripts/SetupCMSSWPset.py b/src/python/WMCore/WMRuntime/Scripts/SetupCMSSWPset.py index 52c4aae37a..5d263ca0c2 100644 --- a/src/python/WMCore/WMRuntime/Scripts/SetupCMSSWPset.py +++ b/src/python/WMCore/WMRuntime/Scripts/SetupCMSSWPset.py @@ -11,7 +11,7 @@ import os import pickle import socket - +from pprint import pformat from PSetTweaks.PSetTweak import PSetTweak from PSetTweaks.WMTweak import makeJobTweak, makeOutputTweak, makeTaskTweak, resizeResources from Utils.Utilities import decodeBytesToUnicode, encodeUnicodeToBytes @@ -705,11 +705,10 @@ def __call__(self): raise ex # Check process.source exists - self.logger.info("Debug: Self.process") - self.logger.info(dir(self.process)) if getattr(self.process, "source", None) is None and getattr(self.process, "_Process__source", None) is None: - msg = "Error in CMSSW PSet: process is missing attribute 'source'" - msg += " or process.source is defined with None value." + msg = "Error in CMSSW PSet: process object is either missing or has " + msg += "None value for attributes 'source' and '_Process__source'. " + msg += f"Details of process object are: {pformat(dir(self.process))}" self.logger.error(msg) raise RuntimeError(msg) From c2b22ed810cec27c6e84e62bbd7a6e3f3b20014f Mon Sep 17 00:00:00 2001 From: Alan Malta Rodrigues Date: Fri, 19 Jul 2024 09:42:21 -0400 Subject: [PATCH 2/2] Check if elements from workqueue workRestrictions are indeed Available Check if elements from workqueue workRestrictions are indeed Available fix id property Move logging level from info to warning --- src/python/WMCore/WorkQueue/WorkQueue.py | 10 +++++++--- src/python/WMCore/WorkQueue/WorkQueueBackend.py | 4 ++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/python/WMCore/WorkQueue/WorkQueue.py b/src/python/WMCore/WorkQueue/WorkQueue.py index 967c344f5b..6fe89cb76f 100644 --- a/src/python/WMCore/WorkQueue/WorkQueue.py +++ b/src/python/WMCore/WorkQueue/WorkQueue.py @@ -484,10 +484,14 @@ def _assignToChildQueue(self, queue, *elements): ele['WMBSUrl'] = self.params["WMBSUrl"] workByRequest.setdefault(ele['RequestName'], 0) workByRequest[ele['RequestName']] += 1 - work = self.parent_queue.saveElements(*elements) - self.logger.info("Assigned work to the child queue for:") - for reqName, numElem in viewitems(workByRequest): + self.logger.info("Setting GQE status to 'Negotiating' and assigning to this child queue for:") + for reqName, numElem in workByRequest.items(): self.logger.info(" %d elements for: %s", numElem, reqName) + + work = self.parent_queue.saveElements(*elements) + self.logger.info("GQE successfully saved for:") + for ele in work: + self.logger.info(" %s under GQE id: %s", ele['RequestName'], ele.id) return work def doneWork(self, elementIDs=None, SubscriptionId=None, WorkflowName=None): diff --git a/src/python/WMCore/WorkQueue/WorkQueueBackend.py b/src/python/WMCore/WorkQueue/WorkQueueBackend.py index d3d069f065..7988cddbc3 100644 --- a/src/python/WMCore/WorkQueue/WorkQueueBackend.py +++ b/src/python/WMCore/WorkQueue/WorkQueueBackend.py @@ -521,6 +521,10 @@ def _evalAvailableWork(self, listElems, thresholds, siteJobCounts, if element['RequestName'] in excludeWorkflows: msg = "Skipping aborted/force-completed workflow: %s, work id: %s" self.logger.info(msg, element['RequestName'], element._id) + elif element['Status'] != 'Available': + # Extra safety mechanism, see https://github.com/dmwm/WMCore/pull/12050 + msg = "Skipping element in unwanted status: %s, work id: %s" + self.logger.warning(msg, element['Status'], element._id) else: sortedElements.append(element) sortAvailableElements(sortedElements)