From f7c2286368939a3d7b87237b374ad73be70b1df8 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Thu, 20 Aug 2026 20:10:05 -0400 Subject: [PATCH] Switch Frontier CI from the revoked hackathon QOS to normal The hackathon QOS grant on CFD154 ended, and every Frontier CI job now fails at submission with "Batch job submission failed: Invalid qos specification" (all 12 Frontier jobs in run 32414716605). retry-sbatch correctly classifies this as non-transient, so the failure is immediate. normal is the only QOS remaining on this allocation that permits more than one job at a time -- debug caps at MaxSubmitPU=1, develop and jupyter at MaxJobsPU=1 -- so it is the only one that can run the CI matrix concurrently. Partition, node/task counts, walltimes, and --stepmgr are unchanged. Turnaround does regress: 1-node queue waits over the preceding three days ran a 0.53 min median under hackathon versus 5.1 min median and 5.2 h p90 under normal. Restoring a priority QOS for CI is the real fix; this unblocks the pipeline in the meantime. --- .github/scripts/submit-slurm-job.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/scripts/submit-slurm-job.sh b/.github/scripts/submit-slurm-job.sh index d3f6ddea5b..b28895e3cd 100755 --- a/.github/scripts/submit-slurm-job.sh +++ b/.github/scripts/submit-slurm-job.sh @@ -54,7 +54,12 @@ case "$cluster" in compiler_flag="f" account="CFD154" job_prefix="MFC" - qos="hackathon" + # The hackathon QOS was a temporary grant and is no longer held by + # CFD154; submitting under it now fails outright with "Invalid qos + # specification". "normal" is the only QOS on this allocation without a + # one-job-at-a-time cap, so it is the only one that can run the CI + # matrix concurrently. + qos="normal" # Let each job's slurmstepd broker its own steps instead of routing # every srun through slurmctld. The in-job test suite launches ~1700+ # srun steps per allocation, which congests the Frontier controller. @@ -67,7 +72,7 @@ case "$cluster" in compiler_flag="famd" account="CFD154" job_prefix="MFC" - qos="hackathon" + qos="normal" extra_sbatch="#SBATCH --stepmgr" test_time="01:59:00" bench_time="01:59:00"