diff --git a/DESCRIPTION b/DESCRIPTION index addccb0..1f8ba0a 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,5 +1,5 @@ Package: EpiModelHPC -Version: 2.9.0 +Version: 2.9.1 Date: 2026-07-28 Title: EpiModel Extensions for High-Performance Computing Description: Extension package to EpiModel to run large-scale stochastic network diff --git a/NEWS.md b/NEWS.md index 7ee01f6..f5e7045 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,9 @@ +# EpiModelHPC 2.9.1 + +## BUG FIXES + +- `degen_watch.sh` no longer judges the job it is running inside. The doctor's own job name necessarily matches `PATTERN`, since that is how it is scoped to a campaign, so it sits inside its own task set; and once the process scan widened past `pgrep -x R` in 2.9.0, its own `sleep` and `ssh` processes became visible too. That reads as a task at 0% CPU with nothing in D-state, which is exactly the `hung` signature, so the doctor confirmed itself hung once per sweep and would have requeued itself. Caught on the first campaign to run 2.9.0: ten consecutive confirmations of the doctor's own job id, spared only because its restart counter already sat at `MAX_RESTARTS`. Any task whose `jobid` equals `SLURM_JOB_ID` is now skipped before classification. + # EpiModelHPC 2.9.0 ## NEW FEATURES diff --git a/inst/hpc_doctor/degen_watch.sh b/inst/hpc_doctor/degen_watch.sh index 4294e19..f3bc14d 100755 --- a/inst/hpc_doctor/degen_watch.sh +++ b/inst/hpc_doctor/degen_watch.sh @@ -141,6 +141,16 @@ while read -r node rest; do jid=$(sed -n 's/.*jobid=\([0-9]*\).*/\1/p' <<< "$rest") med=$(sed -n 's/.*med_cpu=\([0-9]*\).*/\1/p' <<< "$rest") [ -n "${jid:-}" ] && [ -n "${med:-}" ] || continue + # Never judge the job this sweep is running inside. The doctor's own job name + # necessarily matches PATTERN (that is how it is scoped to a campaign), so it + # is inside its own `owned` set, and since the scan widened past `pgrep -x R` + # its own `sleep` and ssh processes are visible too. That reads as a task at + # 0% CPU with nothing in D-state, which is precisely the `hung` signature, and + # the doctor would requeue ITSELF once per sweep. Seen in production the first + # time this shipped: ten consecutive confirmations of jobid 41750366, which + # was the doctor, spared only by its restart counter already sitting at + # MAX_RESTARTS. + if [ -n "${SLURM_JOB_ID:-}" ] && [ "$jid" = "$SLURM_JOB_ID" ]; then continue; fi # Falls back to jid only for a probe old enough to predate `taskid=`; a # same-vintage probe always supplies it. tid=$(sed -n 's/.*taskid=\([0-9_]*\).*/\1/p' <<< "$rest"); tid=${tid:-$jid}