50-main.config

DAEMON_LIST = MASTER, STARTD

# CONDOR_HOST may be overridden by the pilot config file
CONDOR_HOST = cm-1.ospool.osg-htc.org,cm-2.ospool.osg-htc.org
USE_CCB = True
CCB_HEARTBEAT_INTERVAL = 120

FILESYSTEM_DOMAIN = $(HOSTNAME)
UID_DOMAIN = $(HOSTNAME)

LOCAL_DIR=/pilot/condor_local
SPOOL = $(LOCAL_DIR)/spool
LOG = /pilot/log

# now dynamically set in 10-setup-htcondor.sh
#USER_JOB_WRAPPER = /usr/sbin/osgvo-singularity-wrapper

# scitokens
# NOTE: credd will refuse to transfer tokens on a non-encrypted link.
SEC_DEFAULT_ENCRYPTION = REQUIRED
SEC_CREDENTIAL_DIRECTORY = $(LOCAL_DIR)/creds
CREDD_OAUTH_MODE = True
TOKENS = True

SEC_DEFAULT_AUTHENTICATION_METHODS = IDTOKENS,PASSWORD,FS
SEC_CLIENT_AUTHENTICATION_METHODS = IDTOKENS,FS

ALLOW_ADMINISTRATOR = condor_pool@*/*

use feature : GPUs
# dynamic slots
SLOT_TYPE_1 = cpus=100%,gpus=100%,disk=100%,swap=100%
SLOT_TYPE_1_PARTITIONABLE = TRUE
NUM_SLOTS = 1
NUM_SLOTS_TYPE_1 = 1

# make it look like a gwms glidein
GLIDEIN_Country = "US"
GLIDEIN_Site = "Docker"
GLIDEIN_ResourceName = "Docker"

GLIDEIN_ContainerTag = "@CONTAINER_TAG@"

STARTD_ATTRS = $(STARTD_ATTRS) GLIDEIN_Country GLIDEIN_Site GLIDEIN_ResourceName \
               WorkerGroupName IsOsgVoContainer GLIDEIN_ContainerTag \
               GLIDECLIENT_Group \
               Is_ITB_Site \
               START_EXTRA OSG_PROJECT_RESTRICTION

# IsBlackHole is set in "additional-htcondor-config"
# HasExcessiveLoad is set in "additional-htcondor-config"

# only accept new work for a while, for validated nodes and jobs with project name set
# On GPU nodes, CPU only jobs are only allowed after all GPUs are in use
#
# Some attributes used (OSG_NODE_VALIDATED, IsBlackHole, HasExcessiveLoad, ...) might
# not be used/defined by all VOs, so we explicitly check if they are defined before
# using them as part of the START expression.
#
# For GPU containers, we check if we can fit some CPU-only jobs with RoomForCPUOnlyJobs
START = (time() < GLIDEIN_ToRetire) && \
        (ifThenElse(TARGET.JobDurationCategory =?= "Long", GLIDEIN_ToDie - time() > 144000, True)) && \
        (isUndefined(MY.OSG_NODE_VALIDATED) || MY.OSG_NODE_VALIDATED) && \
        (isUndefined(IsBlackHole) || IsBlackHole == False) && \
        (isUndefined(HasExcessiveLoad) || HasExcessiveLoad == False) && \
        ((DESIRED_Sites=?=undefined) || stringListMember(GLIDEIN_Site,DESIRED_Sites,",")) && \
        ((UNDESIRED_Sites=?=undefined) || !stringListMember(GLIDEIN_Site,UNDESIRED_Sites,",")) && \
        (!isUndefined(TARGET.ProjectName)) && \
        ((MY.GPUs ?: 0) == 0 || \
           (TARGET.RequestGPUs ?: 0) > 0 || \
           RoomForCPUOnlyJobs is Undefined || RoomForCPUOnlyJobs) && \
        (isUndefined(TARGET.SingularityImage) || MY.SINGULARITY_START_CLAUSE) && \
        (TARGET.Want_MPI =!= True || MY.Has_MPI =?= True) && \
        (START_EXTRA ?: true) && (OSG_PROJECT_RESTRICTION ?: true)

# Prefer certain types of jobs, for example GPU jobs if they can fit
RANK = (TARGET.RequestGPUs ?: 0)

# Include the osgvo-docker-pilot configuration file, if it exists.
include ifexist: $ENV(PILOT_CONFIG_FILE)

#######################################################################
#
# shutdown if we are out of jobs
#

# when the work is complete, shut down the startd
STARTD_NOCLAIM_SHUTDOWN = $(ACCEPT_IDLE_MINUTES) * 60

# Have the master exit if the startd isn't around and it's been given more
# than sixty seconds to show up.  (We could avoid having a timeout if we
# were sure that START_StartTime was undefined before its first start.)
MASTER.DAEMON_SHUTDOWN_FAST = ((STARTD_StartTime == 0) && ((CurrentTime - DaemonStartTime) > 60)) || (time() > GLIDEIN_ToDie)

# callout to a script when the master exits
DEFAULT_MASTER_SHUTDOWN_SCRIPT = /usr/local/sbin/master_shutdown.sh

# Identify pilot container jobs for gratia-probe detection
IsOsgVoContainer = True

# All daemons should log to syslog instead of to their corresponding files.  In this
# container, syslog is configured to echo all logs to stdout (which the container runtime
# should pick up).
MAX_DEFAULT_LOG = 0
ALL_DEBUG=D_CAT,D_SUB_SECOND,D_PID