From 2620b321fb050ba0a3e0f072e85fe916af09f599 Mon Sep 17 00:00:00 2001
From: Mats Rynge <mats@rynge.net>
Date: Wed, 13 Dec 2023 12:10:31 -0800
Subject: [PATCH 1/3] Replaced CPUJOB_ON_GPUSLOT with the new
 RoomForCPUOnlyJobs

---
 10-setup-htcondor.sh | 8 --------
 50-main.config       | 6 +++++-
 Dockerfile           | 3 ---
 3 files changed, 5 insertions(+), 12 deletions(-)

diff --git a/10-setup-htcondor.sh b/10-setup-htcondor.sh
index 8aab54a..ac27aa3 100755
--- a/10-setup-htcondor.sh
+++ b/10-setup-htcondor.sh
@@ -501,14 +501,6 @@ if [ "x$OSG_SQUID_LOCATION" != "x" ]; then
     export http_proxy="$OSG_SQUID_LOCATION"
 fi
 
-# some admins prefer to reserve gpu slots for gpu jobs, others
-# want to run cpu jobs if there are no gpu jobs available
-if is_true "$ALLOW_CPUJOB_ON_GPUSLOT"; then
-    echo "CPUJOB_ON_GPUSLOT = True" >> "$PILOT_CONFIG_FILE"
-else
-    echo "CPUJOB_ON_GPUSLOT = ifThenElse(MY.TotalGPUs > 0 && MY.GPUs > 0, TARGET.RequestGPUs > 0, True)" >> "$PILOT_CONFIG_FILE"
-fi
-
 cat >$LOCAL_DIR/user-job-wrapper.sh <<EOF
 #!/bin/bash
 set -e
diff --git a/50-main.config b/50-main.config
index 74d890a..8e2cb03 100644
--- a/50-main.config
+++ b/50-main.config
@@ -56,6 +56,8 @@ STARTD_ATTRS = $(STARTD_ATTRS) GLIDEIN_Country GLIDEIN_Site GLIDEIN_ResourceName
 # Some attributes used (OSG_NODE_VALIDATED, IsBlackHole, HasExcessiveLoad, ...) might
 # not be used/defined by all VOs, so we explicitly check if they are defined before
 # using them as part of the START expression.
+#
+# For GPU containers, we check if we can fit some CPU-only jobs with RoomForCPUOnlyJobs
 START = (time() < GLIDEIN_ToRetire) && \
         (ifThenElse(TARGET.JobDurationCategory =?= "Long", GLIDEIN_ToDie - time() > 144000, True)) && \
         (isUndefined(MY.OSG_NODE_VALIDATED) || MY.OSG_NODE_VALIDATED) && \
@@ -64,7 +66,9 @@ START = (time() < GLIDEIN_ToRetire) && \
         ((DESIRED_Sites=?=undefined) || stringListMember(GLIDEIN_Site,DESIRED_Sites,",")) && \
         ((UNDESIRED_Sites=?=undefined) || !stringListMember(GLIDEIN_Site,UNDESIRED_Sites,",")) && \
         (!isUndefined(TARGET.ProjectName)) && \
-        ($(CPUJOB_ON_GPUSLOT)) && \
+        (MY.GPUs is Undefined || MY.GPUs == 0 || \
+           TARGET.RequestGPUs is Undefined || TARGET.RequestGPUs > 0 || \
+           RoomForCPUOnlyJobs is Undefined || RoomForCPUOnlyJobs) && \
         (isUndefined(TARGET.SingularityImage) || MY.SINGULARITY_START_CLAUSE) && \
         (TARGET.Want_MPI =!= True || MY.Has_MPI =?= True) && \
         (START_EXTRA ?: true) && (OSG_PROJECT_RESTRICTION ?: true)
diff --git a/Dockerfile b/Dockerfile
index aa9c8f2..8662d42 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -209,9 +209,6 @@ ENV SINGULARITY_BIND_EXTRA=
 # Additional restrictions for your START expression
 ENV GLIDEIN_Start_Extra=
 
-# Allow CPU jobs on GPU slots if there are GPUs left
-ENV ALLOW_CPUJOB_ON_GPUSLOT=false
-
 # Use the prepare-job-hook to run Singularity jobs
 ENV CONTAINER_PILOT_USE_JOB_HOOK=true
 

From fd4a3d600b697cd29b58c3f9bf9efda3bd3654c4 Mon Sep 17 00:00:00 2001
From: Mats Rynge <mats@rynge.net>
Date: Wed, 13 Dec 2023 14:23:14 -0800
Subject: [PATCH 2/3] Update 50-main.config

Co-authored-by: Matyas Selmeci <matyas@cs.wisc.edu>
---
 50-main.config | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/50-main.config b/50-main.config
index 8e2cb03..a59ec4e 100644
--- a/50-main.config
+++ b/50-main.config
@@ -67,7 +67,7 @@ START = (time() < GLIDEIN_ToRetire) && \
         ((UNDESIRED_Sites=?=undefined) || !stringListMember(GLIDEIN_Site,UNDESIRED_Sites,",")) && \
         (!isUndefined(TARGET.ProjectName)) && \
         (MY.GPUs is Undefined || MY.GPUs == 0 || \
-           TARGET.RequestGPUs is Undefined || TARGET.RequestGPUs > 0 || \
+           (TARGET.RequestGPUs ?: 0) > 0 || \
            RoomForCPUOnlyJobs is Undefined || RoomForCPUOnlyJobs) && \
         (isUndefined(TARGET.SingularityImage) || MY.SINGULARITY_START_CLAUSE) && \
         (TARGET.Want_MPI =!= True || MY.Has_MPI =?= True) && \

From ace88c58bb64a4c34e8d820a5e62acf17688fc81 Mon Sep 17 00:00:00 2001
From: Mats Rynge <mats@rynge.net>
Date: Wed, 13 Dec 2023 14:23:22 -0800
Subject: [PATCH 3/3] Update 50-main.config

Co-authored-by: Matyas Selmeci <matyas@cs.wisc.edu>
---
 50-main.config | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/50-main.config b/50-main.config
index a59ec4e..c16798e 100644
--- a/50-main.config
+++ b/50-main.config
@@ -66,7 +66,7 @@ START = (time() < GLIDEIN_ToRetire) && \
         ((DESIRED_Sites=?=undefined) || stringListMember(GLIDEIN_Site,DESIRED_Sites,",")) && \
         ((UNDESIRED_Sites=?=undefined) || !stringListMember(GLIDEIN_Site,UNDESIRED_Sites,",")) && \
         (!isUndefined(TARGET.ProjectName)) && \
-        (MY.GPUs is Undefined || MY.GPUs == 0 || \
+        ((MY.GPUs ?: 0) == 0 || \
            (TARGET.RequestGPUs ?: 0) > 0 || \
            RoomForCPUOnlyJobs is Undefined || RoomForCPUOnlyJobs) && \
         (isUndefined(TARGET.SingularityImage) || MY.SINGULARITY_START_CLAUSE) && \