From 85754cf647123df2e6392e91af5c00478dab2a7f Mon Sep 17 00:00:00 2001 From: trtllm-agent <296075020+trtllm-agent@users.noreply.github.com> Date: Thu, 6 Aug 2026 10:21:19 -0700 Subject: [PATCH] [nvbugs/6561778][fix] Fence all Slurm ranks before launching pytest The only install lock (slurm_install.sh) lives under $resourcePathNode, which is /tmp -- node-local, and a per-step tmpfs under pyxis -- so its wait loop fences just the $SLURM_LOCALID peers of one node and a node can never observe another node's lock. Nothing then stops slurm_run.sh from reaching `eval $pytestCommand` on one node while another is still installing, and the per-rank work in between skews the ranks further: non-zero ranks cover rank 0's coverage-config write with a blind `sleep 30`, and slurm_setup_runtime_env shells out to pip3. Pytest's first action is `import tensorrt_llm`, whose module-scope MPI collective must be entered by every rank. Under --mpi=pmix, which is added exactly when nodeCount > 1, that collective has a 300s fence timeout, so a node whose pip3 install stalls (up to the 2700s retry budget) makes the collective abort every rank rather than merely run late. The ranks die between pytest setup and teardown, which leaves the nodeid in unfinished_test.txt and makes generate_timeout_xml.py synthesize the "Test terminated unexpectedly" this bug reports -- there is no traceback for the test body. PMIX_MCA_gds=hash does not mitigate this: a fence that times out never exchanges the modex regardless of GDS mode, and the pml_ucx errors seen alongside it are downstream of that same missing exchange. Add a marker barrier on the shared $jobWorkspace immediately before `eval $pytestCommand`. It counts SLURM_NTASKS rank markers rather than nodes, so the fenced set is exactly the set that enters the aborting collective, and placing it after the block that wipes SLURM_* keeps it a no-op for single-node and disaggregated benchmark/server runs, which reach it with SLURM_NTASKS unset. The marker directory is keyed per job and per step because $jobWorkspace outlives a step, so a later step must not be released by an earlier one's markers. The wait is bounded above the 2700s pip3 budget so a genuinely dead rank fails the stage with a clear message instead of hanging until the partition walltime. The barrier is byte-identical to the one reviewed on the sibling attribution of this same defect (6541343), so whichever lands first collapses the other's hunk on rebase. Also un-waive this bug's test, whose body was already healthy: the reproduce log passes all three accuracy phases (MMLU 87.013 vs 85.513, GSM8K 95.375 vs 92.217, CnnDailymail rouge1 30.271 vs 26.716) and fails only on an fsspecIO thread leaked by HuggingFace's downloader, an environment artifact unrelated to the launch defect. Signed-off-by: trtllm-agent <296075020+trtllm-agent@users.noreply.github.com> --- jenkins/scripts/slurm_run.sh | 55 +++++++++++++++++++++++++ tests/integration/test_lists/waives.txt | 1 - 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/jenkins/scripts/slurm_run.sh b/jenkins/scripts/slurm_run.sh index 25622daff391..a90dda49262e 100755 --- a/jenkins/scripts/slurm_run.sh +++ b/jenkins/scripts/slurm_run.sh @@ -72,6 +72,61 @@ if [ "${SLURM_JOB_NUM_NODES:-1}" -eq 1 ] || \ done fi +# The install lock in slurm_install.sh lives under $resourcePathNode (/tmp), so it +# is node-local: its wait loop only fences $SLURM_LOCALID peers on the same node, +# and a node can never observe another node's lock. Nothing else stops one node +# from reaching `eval $pytestCommand` below while another is still installing, and +# the per-rank work above skews the ranks further (non-zero ranks cover the +# coverage-config write with a blind `sleep 30`, and slurm_setup_runtime_env shells +# out to pip3). Pytest's first action is `import tensorrt_llm`, whose module-scope +# MPI collective must be entered by every rank; under --mpi=pmix -- added exactly +# when nodeCount > 1 -- that collective has a 300s fence timeout, so the skew +# aborts every rank instead of merely running late. Fence every rank on the shared +# $jobWorkspace so they enter pytest together. +slurm_wait_all_ranks() { + local numRanks="${SLURM_NTASKS:-1}" + if [ "$numRanks" -le 1 ] || [ -z "${jobWorkspace:-}" ]; then + return 0 + fi + + # Keyed per job *and* per step: $jobWorkspace outlives a single step, so + # markers from another job, or from an earlier step of this job, must not + # satisfy the count. Slurm assigns one step id per step across all of its + # nodes, so every rank of a step agrees on this path. + local readyDir="$jobWorkspace/run_ready_job_${SLURM_JOB_ID:-local}_step_${SLURM_STEP_ID:-0}" + mkdir -p "$readyDir" + touch "$readyDir/rank_${SLURM_PROCID}.ready" + + # Bounded so a dead rank fails the stage loudly instead of hanging until the + # partition walltime kills it; the ceiling exceeds the 2700s pip3 retry budget + # in slurm_install.sh so a merely slow rank still releases the barrier. + local timeoutSecs=3600 + local deadline=$((SECONDS + timeoutSecs)) + local markers ready + while true; do + # Counted with a glob rather than `ls | wc -l`: under `set -Eeuo pipefail` a + # failing `ls` propagates into the assignment and fires the ERR trap. The + # touch above guarantees at least one match, so no nullglob is needed. + markers=("$readyDir"/*.ready) + ready=${#markers[@]} + if [ "$ready" -ge "$numRanks" ]; then + return 0 + fi + if [ "$SECONDS" -ge "$deadline" ]; then + echo "ERROR: rank ${SLURM_PROCID} timed out after ${timeoutSecs}s waiting for" \ + "all $numRanks ranks to be ready; ready: $ready/$numRanks" + return 1 + fi + # One rank reports progress; all of them would spam the log every 10s. + if [ "$SLURM_PROCID" -eq 0 ]; then + echo "(Waiting for all $numRanks ranks to be ready) ready: $ready/$numRanks" + fi + sleep 10 + done +} + +slurm_wait_all_ranks + # Turn off "exit on error" so the following lines always run set +e diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 71b5595714c3..01ad0aee0c99 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -21,7 +21,6 @@ accuracy/test_llm_api_autodeploy.py::TestQwen3_5_397B_MoE::test_bf16_small[4] SK accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_fp8_blockscale[throughput] SKIP (https://nvbugs/6561775) accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_fp8_blockscale[throughput_mtp] SKIP (https://nvbugs/6428101) accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_fp8_blockscale[throughput_mtp_trtllm] SKIP (https://nvbugs/6426868) -accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[latency] SKIP (https://nvbugs/6561778) accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[latency_adp_lmtp] SKIP (https://nvbugs/6561777) accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput_pp4_mtp] SKIP (https://nvbugs/6481323) accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_dsa_host_cache_offload[host_cache_offload_mtp1] SKIP (https://nvbugs/6384357)