Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Batch update: copy max_* params to resourcelimit directive #758

Merged
merged 17 commits into from
Oct 7, 2024
Merged

Batch update: copy max_* params to resourcelimit directive #758

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
32a6072
First batch (automated) of copying max_* params to resourceLimits dir…
jfy133 Sep 23, 2024
c5822f8
Remove empty resourceLimit entries
jfy133 Sep 23, 2024
7ac14ce
Merge branch 'master' into batch-resourcelimit-update
jfy133 Sep 24, 2024
76cd1d9
Manually update more complicated configs for resourceLimits
jfy133 Sep 24, 2024
fe18e95
Merge branch 'master' into batch-resourcelimit-update
jfy133 Sep 25, 2024
79899c3
Remove unused check_max() functions from configs
jfy133 Sep 25, 2024
0870776
Remove check_max() calls from configs using both in a simple manner
jfy133 Sep 25, 2024
00e757f
Re-add the task.attempt to the old check_max calls in instituional pr…
jfy133 Sep 25, 2024
7461bd1
Update conf/crukmi.config
jfy133 Sep 25, 2024
3ec677e
Apply suggestions from code review
jfy133 Oct 4, 2024
74a4a1e
Don't assume time inheritance in resourceLimits for arcc
jfy133 Oct 4, 2024
85905fa
Merge branch 'batch-resourcelimit-update' of github.com:jfy133/nf-cor…
jfy133 Oct 4, 2024
c368a77
Remove checkmax call in utd_ganymede
jfy133 Oct 4, 2024
9e5265b
Merge branch 'master' into batch-resourcelimit-update
jfy133 Oct 4, 2024
5b12fc2
Formatting
jfy133 Oct 4, 2024
b8f2140
Remove check_max calls and copied function, and remove duplicated blo…
jfy133 Oct 4, 2024
4264245
Fix STJude profile
jfy133 Oct 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions conf/abims.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,11 @@ singularity {
}

process {
resourceLimits = [
memory: 750.GB,
cpus: 200,
time: 30.d
]
executor = 'slurm'
queue = { task.memory <= 250.GB ? (task.time <= 24.h ? 'fast' : 'long') : 'bigmem' }
}
Expand Down
5 changes: 5 additions & 0 deletions conf/adcra.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,11 @@ singularity {
}

process {
resourceLimits = [
memory: 128.GB,
cpus: 16,
time: 120.h
]
scratch = true
queue = 'unlimit'
queueStatInterval = '10 min'
Expand Down
16 changes: 13 additions & 3 deletions conf/alice.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,13 @@ params {
}

process {
executor = 'slurm'
queue = { task.time < 3.h ? 'cpu-short' : task.time < 24.h ? 'cpu-medium' : 'cpu-long' }
resourceLimits = [
memory: 240.GB,
cpus: 24,
time: 168.h
]
executor = 'slurm'
queue = { task.time < 3.h ? 'cpu-short' : task.time < 24.h ? 'cpu-medium' : 'cpu-long' }
}

singularity {
Expand All @@ -30,7 +35,12 @@ profiles {
max_time = 336.h
}
process {
queue = 'mem'
resourceLimits = [
memory: 2.TB,
cpus: 24,
time: 336.h
]
queue = 'mem'
}
}
debug {
Expand Down
5 changes: 5 additions & 0 deletions conf/apollo.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,11 @@ params {
}

process {
resourceLimits = [
memory: 1500.GB,
cpus: 72,
time: 500.h
]
executor = 'slurm'
queue = 'all'
beforeScript = 'module load singularity'
Expand Down
53 changes: 45 additions & 8 deletions conf/arcc.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,11 @@ params {
}

process {
resourceLimits = [
memory: 64.GB,
cpus: 16,
time: 1.h
]
executor = 'slurm'
scratch = false
clusterOptions = "--account=${System.getenv('SBATCH_ACCOUNT')}"
Expand All @@ -38,7 +43,11 @@ profiles {
}

process {
queue = 'beartooth'
resourceLimits = [
memory: 256.GB,
cpus: 46
]
queue = 'beartooth'
}
}

Expand All @@ -53,7 +62,11 @@ profiles {
}

process {
queue = 'beartooth-bigmem'
resourceLimits = [
memory: 512.GB,
cpus: 56
]
queue = 'beartooth-bigmem'
}
}

Expand All @@ -68,7 +81,11 @@ profiles {
}

process {
queue = 'beartooth-hugemem'
resourceLimits = [
memory: 1024.GB,
cpus: 56
]
queue = 'beartooth-hugemem'
}
}

Expand All @@ -83,7 +100,11 @@ profiles {
}

process {
queue = 'moran'
resourceLimits = [
memory: 64.GB,
cpus: 16
]
queue = 'moran'
}
}

Expand All @@ -98,7 +119,11 @@ profiles {
}

process {
queue = 'teton'
resourceLimits = [
memory: 128.GB,
cpus: 32
]
queue = 'teton'
}
}

Expand All @@ -113,7 +138,11 @@ profiles {
}

process {
queue = 'teton-cascade'
resourceLimits = [
memory: 768.GB,
cpus: 40
]
queue = 'teton-cascade'
}
}

Expand All @@ -128,7 +157,11 @@ profiles {
}

process {
queue = 'teton-hugemem'
resourceLimits = [
memory: 1024.GB,
cpus: 32
]
queue = 'teton-hugemem'
}
}

Expand All @@ -143,7 +176,11 @@ profiles {
}

process {
queue = 'teton-knl'
resourceLimits = [
memory: 384.GB,
cpus: 72
]
queue = 'teton-knl'
}
}
}
5 changes: 5 additions & 0 deletions conf/bih.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,11 @@ params {
}

process {
resourceLimits = [
memory: 340.GB,
cpus: 64,
time: 7.d
]
executor = 'slurm'
maxRetries = 2
clusterOptions = "--export=ALL"
Expand Down
5 changes: 5 additions & 0 deletions conf/binac.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,11 @@ singularity {
}

process {
resourceLimits = [
memory: 1000.GB,
cpus: 28,
time: 168.h
]
executor = 'pbs'
queue = { task.memory >= 128.GB ? 'smp' : task.time <= 20.m ? 'tiny' : task.time > 48.h ? 'long' : 'short' }
}
Expand Down
5 changes: 5 additions & 0 deletions conf/biohpc_gen.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,11 @@ params {
}

process {
resourceLimits = [
memory: 3.TB,
cpus: 80,
time: 14.d
]
executor = 'slurm'
queue = { task.memory <= 1536.GB ? (task.time > 2.d || task.memory > 384.GB ? 'biohpc_gen_production' : 'biohpc_gen_normal') : 'biohpc_gen_highmem' }
clusterOptions = '--clusters=biohpc_gen'
Expand Down
5 changes: 5 additions & 0 deletions conf/biowulf.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,11 @@ env {


process {
resourceLimits = [
memory: 224,
cpus: 32,
time: 72
]
executor = 'slurm'
maxRetries = 1

Expand Down
5 changes: 5 additions & 0 deletions conf/bluebear.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,11 @@ singularity {
}

process {
resourceLimits = [
memory: 512.GB,
cpus: 112,
time: 10.d
]
executor = 'slurm'
clusterOptions = {
"--account ${System.getenv("SLURM_JOB_ACCOUNT")} " + "--qos ${task.time <= 10.m ? 'bbshort' : 'bbdefault'}"
Expand Down
5 changes: 5 additions & 0 deletions conf/cambridge.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,11 @@ singularity {
}

process {
resourceLimits = [
memory: 192.GB,
cpus: 56,
time: 12.h
]
executor = 'slurm'
clusterOptions = "-A ${params.project} -p ${params.partition}"
}
5 changes: 5 additions & 0 deletions conf/cbe.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,11 @@ params {
}

process {
resourceLimits = [
memory: 1800.GB,
cpus: 36,
time: 14.d
]
executor = 'slurm'
queue = { task.memory <= 120.GB ? 'c' : 'm' }
module = ['build-env/.f2021', 'build-env/f2021', 'anaconda3/2021.11']
Expand Down
5 changes: 5 additions & 0 deletions conf/ccga_dx.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,11 @@ executor {
}

process {
resourceLimits = [
memory: 250.GB,
cpus: 20,
time: 240.h
]

// Global process config
executor = 'slurm'
Expand Down
5 changes: 5 additions & 0 deletions conf/ccga_med.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,11 @@ executor {
}

process {
resourceLimits = [
memory: 250.GB,
cpus: 24,
time: 120.h
]

// Global process config
executor = 'slurm'
Expand Down
5 changes: 5 additions & 0 deletions conf/cedars.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,11 @@ executor {
}

process {
resourceLimits = [
memory: 90.GB,
cpus: 10,
time: 240.h
]
penv = 'smp'
beforeScript = """
module load 'singularity/3.6.0'
Expand Down
5 changes: 5 additions & 0 deletions conf/ceres.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,11 @@ singularity {
}

process {
resourceLimits = [
memory: 640.GB,
cpus: 36,
time: 60.d
]
executor = 'slurm'
scratch = true
queue = {
Expand Down
5 changes: 5 additions & 0 deletions conf/cfc.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,11 @@ singularity {
}

process {
resourceLimits = [
memory: 1992.GB,
cpus: 128,
time: 168.h
]
executor = 'slurm'
queue = 'qbic'
scratch = 'true'
Expand Down
5 changes: 5 additions & 0 deletions conf/cfc_dev.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,11 @@ singularity {
}

process {
resourceLimits = [
memory: 1992.GB,
cpus: 128,
time: 168.h
]
executor = 'slurm'
queue = 'qbic'
scratch = 'true'
Expand Down
5 changes: 5 additions & 0 deletions conf/cheaha.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,11 @@ singularity {
}

process {
resourceLimits = [
memory: 750.GB,
cpus: 128,
time: 150.h
]
executor = 'slurm'
queue = { task.memory <= 50.GB ? (task.time <= 2.h ? 'express' : task.time <= 12.h ? 'short' : task.time <= 50.h ? 'medium' : 'long') : (task.time <= 50.h ? 'largemem' : 'largemem-long')}
maxRetries = 3
Expand Down
5 changes: 5 additions & 0 deletions conf/computerome.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,11 @@ singularity {
}

process {
resourceLimits = [
memory: 1500.GB,
cpus: 40,
time: 168.h
]
beforeScript = "module load tools singularity/3.8.0; export _JAVA_OPTIONS=-Djava.io.tmpdir=${params.cache_dir}"
executor = 'pbs'
queueSize = 2000
Expand Down
5 changes: 5 additions & 0 deletions conf/create.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,5 +13,10 @@ singularity {
}

process {
resourceLimits = [
memory: 1024.GB,
cpus: 128,
time: 48.h
]
executor = 'slurm'
}
5 changes: 5 additions & 0 deletions conf/crick.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,11 @@ singularity {
}

process {
resourceLimits = [
memory: 2.TB,
cpus: 256,
time: 168.h
]
executor = 'slurm'
}

Expand Down
Loading
Loading