From 8eda3618f629f7c029bce601ae01f2751b88b9c1 Mon Sep 17 00:00:00 2001 From: tianyuz-nv Date: Mon, 10 Aug 2026 00:25:35 -0700 Subject: [PATCH 1/3] [None][test] Replace disaggregated DWDP accuracy tests with aggregated coverage DWDP accuracy was gated by three disaggregated-serving tests that are currently waived on GB200 and B200, so the feature has no effective CI coverage. Those tests exercise DWDP through the disaggregated KV cache transceiver, which makes them sensitive to per-cluster UCX transport configuration rather than to DWDP itself. Add an aggregated equivalent instead. A single instance running attention DP satisfies the invariant DWDP relies on -- every rank is a complete model replica owning one expert slice -- because Mapping.dp_size == tp_size there and attention is replicated rather than tensor-sharded. Relax the DWDP gate in create_py_executor accordingly: tp_size > 1 is now accepted when attention DP is enabled, and real tensor parallelism is still rejected with an explicit error. Mapping already forces moe_tp = moe_ep = 1 whenever dwdp_size > 1, so expert weights stay unsharded and ConfigurableMoE selects no MoE communication strategy on this path. The new tests run at dwdp_size=4 rather than the 2 the disaggregated tests used: aggregated serving has no generation server, so the whole allocation goes to DWDP peers. Three remote peers per rank also make contention_opt meaningful, since it interleaves prefetch slices across peers -- with a single remote peer that path was degenerate. Retire the disaggregated tests from the CI lists and drop their now-dead waives, but keep the file in tree as a manual reproduction of the disaggregated DWDP path, with a note on the UCX_TLS setting to check first. Signed-off-by: tianyuz-nv --- .../_torch/pyexecutor/py_executor_creator.py | 30 +++++- .../defs/accuracy/test_dwdp_aggregated.py | 98 +++++++++++++++++++ .../test_dwdp_disaggregated_serving.py | 14 +++ .../test_lists/qa/llm_function_core.txt | 6 +- .../test-db/l0_gb200_multi_gpus.yml | 6 +- tests/integration/test_lists/waives.txt | 3 - 6 files changed, 146 insertions(+), 11 deletions(-) create mode 100644 tests/integration/defs/accuracy/test_dwdp_aggregated.py diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py b/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py index 0b89c079b3bc..352f02147755 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py @@ -536,10 +536,36 @@ def create_py_executor( ) logger.info("ATTENTION RUNTIME FEATURES: ", attn_runtime_features) - # Initialize DWDP Manager (only for context workers in disaggregated serving) + # Initialize DWDP Manager. + # + # DWDP requires every global MPI rank to be a complete, unsharded model + # replica owning one expert slice: `dwdp_rank = global_mpi_rank() % + # dwdp_size` is only a meaningful identity under that bijection, and the + # layout installed by `_init_dwdp_expert_layout` assumes expert weights are + # not additionally tensor-sharded. Two deployments satisfy this: + # * disaggregated serving -- each context worker is its own TP=1 instance; + # DWDP peers are separate instances joined by the global MPI world. + # * aggregated serving with attention DP -- one instance in which + # `mapping.dp_size == tp_size` (see Mapping.dp_size), so attention is + # replicated and data-parallel rather than tensor-sharded and each rank + # is still a full replica. Mapping already forces `moe_tp = moe_ep = 1` + # whenever `dwdp_size > 1`, so expert weights stay unsharded and + # ConfigurableMoE selects no MoE communication strategy. + # Real tensor parallelism (tp_size > 1 without attention DP) is rejected: a + # rank would be a shard of a replica, breaking both the rank-to-worker + # bijection and the unsharded-expert assumption. dwdp_manager: Optional[DwdpManager] = None if llm_args.dwdp_config is not None: - assert mapping.tp_size == 1 and llm_args.dwdp_config.dwdp_size > 1, "DWDP requires TP=1 and dwdp_size > 1" + if llm_args.dwdp_config.dwdp_size <= 1: + raise ValueError( + f"DWDP requires dwdp_size > 1, got {llm_args.dwdp_config.dwdp_size}." + ) + if mapping.tp_size > 1 and not mapping.enable_attention_dp: + raise ValueError( + "DWDP requires each rank to be a complete model replica: use " + "tp_size=1 (disaggregated context worker) or " + "enable_attention_dp=True (aggregated serving), but got " + f"tp_size={mapping.tp_size} with enable_attention_dp=False.") dwdp_manager = DwdpManager(config=llm_args.dwdp_config, dist=dist, mapping=mapping) diff --git a/tests/integration/defs/accuracy/test_dwdp_aggregated.py b/tests/integration/defs/accuracy/test_dwdp_aggregated.py new file mode 100644 index 000000000000..c3448f43406f --- /dev/null +++ b/tests/integration/defs/accuracy/test_dwdp_aggregated.py @@ -0,0 +1,98 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""DWDP accuracy tests in aggregated serving. + +DWDP was originally built for the context phase of disaggregated serving, where +each context worker is its own TP=1 instance and the workers are joined into one +MPI world by ``trtllm-serve disaggregated_mpi_worker``. The same invariant -- +every rank is a complete model replica owning one expert slice -- also holds for +a single aggregated instance running attention DP, because ``Mapping.dp_size == +tp_size`` there and attention is replicated rather than tensor-sharded. + +Running DWDP aggregated keeps the expert-sharing paths under test while removing +the disaggregated KV cache transceiver from the picture, so these tests do not +depend on the UCX transport configuration of the cluster they run on. +""" + +import pytest + +from tensorrt_llm import LLM +from tensorrt_llm.llmapi import KvCacheConfig, MoeConfig +from tensorrt_llm.llmapi.llm_args import DwdpConfig + +from ..conftest import llm_models_root, skip_post_blackwell_ultra, skip_pre_blackwell +from .accuracy_core import GSM8K, LlmapiAccuracyTestHarness + +# DeepSeek-V3-Lite has 72 routed experts, partitioned across ``DWDP_SIZE`` +# workers. Mode A is the uniform partition (``size == stride == 72 // +# DWDP_SIZE``); Mode B uses ``size > stride`` so adjacent peer ranges overlap, +# and ``(DWDP_SIZE - 1) * stride + size == 72`` must hold exactly. +# +# DWDP_SIZE is 4 rather than the 2 the disaggregated tests used: aggregated +# serving has no generation server, so the whole allocation goes to DWDP peers. +# Three remote peers per rank instead of one is also what makes +# ``contention_opt`` meaningful, since it interleaves prefetch slices across +# peers to spread them over several NVLink links. +DWDP_SIZE = 4 + + +class TestDwdpAggDeepSeekV3Lite(LlmapiAccuracyTestHarness): + MODEL_NAME = "deepseek-ai/DeepSeek-V3-Lite" + + @pytest.mark.skip_less_device(DWDP_SIZE) + @skip_pre_blackwell + @skip_post_blackwell_ultra + @pytest.mark.parametrize( + "num_experts_per_worker,num_prefetch_experts,contention_opt", + [ + (18, 18, False), + (24, 16, False), + (18, 18, True), + ], + ids=["mode_a_uniform", "mode_b_overlap", "mode_a_uniform_contention_opt"], + ) + def test_dwdp_agg_accuracy(self, num_experts_per_worker, num_prefetch_experts, contention_opt): + dwdp_config = DwdpConfig( + dwdp_size=DWDP_SIZE, + num_groups=1, + num_experts_per_worker=num_experts_per_worker, + num_prefetch_experts=num_prefetch_experts, + contention_opt=contention_opt, + ) + + # Attention DP makes each of the DWDP_SIZE ranks an independent replica + # serving its own requests; DWDP supplies the expert weights a rank does + # not hold locally. Overlap scheduling is not supported with DWDP. + with LLM( + f"{llm_models_root()}/DeepSeek-V3-Lite/nvfp4_moe_only_mtp", + tensor_parallel_size=DWDP_SIZE, + enable_attention_dp=True, + dwdp_config=dwdp_config, + moe_config=MoeConfig(backend="CUTEDSL"), + disable_overlap_scheduler=True, + enable_autotuner=False, + enable_chunked_prefill=False, + cuda_graph_config=None, + max_batch_size=16, + max_num_tokens=8192, + kv_cache_config=KvCacheConfig( + free_gpu_memory_fraction=0.4, + enable_block_reuse=False, + enable_partial_reuse=False, + tokens_per_block=32, + ), + ) as llm: + task = GSM8K(self.MODEL_NAME) + task.evaluate(llm) diff --git a/tests/integration/defs/accuracy/test_dwdp_disaggregated_serving.py b/tests/integration/defs/accuracy/test_dwdp_disaggregated_serving.py index cb98648cfab8..386f9891e193 100644 --- a/tests/integration/defs/accuracy/test_dwdp_disaggregated_serving.py +++ b/tests/integration/defs/accuracy/test_dwdp_disaggregated_serving.py @@ -2,6 +2,20 @@ Separated from test_disaggregated_serving.py to isolate MPI-dependent test infrastructure for easier maintenance. + +NOTE: these tests are intentionally not registered in any CI test list. DWDP +accuracy is gated in CI by test_dwdp_aggregated.py, which exercises the same +expert-sharing paths without the disaggregated KV cache transceiver and is +therefore not exposed to per-cluster transport configuration. This file is kept +as a manual reproduction of the disaggregated DWDP path; run it directly with +pytest. + +When running it manually, check the launcher's UCX settings first: SLURM +enroot/pyxis injects ``UCX_TLS=tcp`` from the host MPI stack on some clusters, +which pins the KV cache transceiver to a transport that can fail there and hang +the run in ``check_gen_transfer_status``. Clear or pin ``UCX_TLS`` for the +cluster before running -- see jenkins/scripts/slurm_env_setup.sh and +examples/disaggregated/slurm/benchmark/start_worker_dwdp.sh. """ import contextlib diff --git a/tests/integration/test_lists/qa/llm_function_core.txt b/tests/integration/test_lists/qa/llm_function_core.txt index 5c48a75d648d..5919c8dbdb37 100644 --- a/tests/integration/test_lists/qa/llm_function_core.txt +++ b/tests/integration/test_lists/qa/llm_function_core.txt @@ -67,9 +67,9 @@ accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_chunked_prefill accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_gen_first accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_gen_first_kv_cache_v1 accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_nixl_backend -accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy -accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy_contention_opt -accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy_mode_b_overlap +accuracy/test_dwdp_aggregated.py::TestDwdpAggDeepSeekV3Lite::test_dwdp_agg_accuracy[mode_a_uniform] +accuracy/test_dwdp_aggregated.py::TestDwdpAggDeepSeekV3Lite::test_dwdp_agg_accuracy[mode_b_overlap] +accuracy/test_dwdp_aggregated.py::TestDwdpAggDeepSeekV3Lite::test_dwdp_agg_accuracy[mode_a_uniform_contention_opt] accuracy/test_epd_disagg_multimodal.py::TestVideoMMEEPD::test_disaggregated_videomme[nemotron_nano_v3_omni_fp8] accuracy/test_epd_disagg_multimodal.py::TestVideoMMEEPD::test_disaggregated_videomme[nemotron_nano_v3_omni_nvfp4] accuracy/test_epd_disagg_multimodal.py::TestVideoMMEEPD::test_disaggregated_videomme[qwen3vl_2b_instruct] diff --git a/tests/integration/test_lists/test-db/l0_gb200_multi_gpus.yml b/tests/integration/test_lists/test-db/l0_gb200_multi_gpus.yml index 044926cd3f11..dceecb3761be 100644 --- a/tests/integration/test_lists/test-db/l0_gb200_multi_gpus.yml +++ b/tests/integration/test_lists/test-db/l0_gb200_multi_gpus.yml @@ -49,9 +49,9 @@ l0_gb200_multi_gpus: - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_eagle3_4gpus[v2_kv_cache-trtllm-one_model-overlap_scheduler] - accuracy/test_llm_api_pytorch_multimodal.py::TestMistralLarge3_675B::test_nvfp4_4gpus[latency_moe_trtllm] TIMEOUT (90) - accuracy/test_disaggregated_serving.py::TestDeepSeekV4Flash::test_auto_dtype TIMEOUT (60) - - accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy - - accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy_contention_opt - - accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy_mode_b_overlap + - accuracy/test_dwdp_aggregated.py::TestDwdpAggDeepSeekV3Lite::test_dwdp_agg_accuracy[mode_a_uniform] + - accuracy/test_dwdp_aggregated.py::TestDwdpAggDeepSeekV3Lite::test_dwdp_agg_accuracy[mode_b_overlap] + - accuracy/test_dwdp_aggregated.py::TestDwdpAggDeepSeekV3Lite::test_dwdp_agg_accuracy[mode_a_uniform_contention_opt] - unittest/_torch/modules/moe/test_moe_comm.py::TestMoEComm::test_moe_comm - unittest/_torch/modules/moe/test_moe_comm.py::TestMoEComm::test_nccl_ep_cuda_graph_replay_uses_updated_routing - unittest/_torch/modules/moe/test_moe_comm.py::TestMoEComm::test_moe_comm_postquant diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index abc96646e567..c478493daa26 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -181,9 +181,6 @@ full:DGX_H100/unittest/llmapi/test_llm_multi_gpu_pytorch.py -m "gpu4" SKIP (http full:DGX_H100/unittest/llmapi/test_llm_multi_gpu_pytorch.py::test_llm_get_stats_pp4[False-False-True] SKIP (https://nvbugs/6618098) full:DGX_H100/unittest/llmapi/test_llm_multi_gpu_pytorch.py::test_tinyllama_logits_processor_tp2pp2 SKIP (https://nvbugs/6618106) full:GB200/accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[ctx_block_reuse_only] SKIP (https://nvbugs/6525893) -full:GB200/accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy SKIP (https://nvbugs/6276923) -full:GB200/accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy_contention_opt SKIP (https://nvbugs/6276923) -full:GB200/accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy_mode_b_overlap SKIP (https://nvbugs/6276923) full:GB200/accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[fp8-4-attn_dp_off-trtllm] SKIP (https://nvbugs/6539942) full:GB200/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False] SKIP (https://nvbugs/6525896) full:GB200/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_bf16_gemm[cuda_graph=True] SKIP (https://nvbugs/6525897) From 71e1417c9484029b119fdba2f5137b5ecfcb8f31 Mon Sep 17 00:00:00 2001 From: tianyuz-nv Date: Wed, 12 Aug 2026 21:54:18 -0700 Subject: [PATCH 2/3] [None][test] Address review feedback on DWDP aggregated accuracy tests Reject pipeline and context parallelism for DWDP. The previous tp_size == 1 assert never covered them, yet a pipeline or context parallel rank is a shard of a replica rather than a replica, and dwdp_rank = global_mpi_rank() % dwdp_size would pair ranks holding different layers as DWDP peers -- wrong expert weights instead of a startup error. Test the tensor-parallel invariant directly as dp_size == tp_size instead of using enable_attention_dp as a proxy for it. The two are equivalent while Mapping.dp_size derives from that flag, but a future partial attention DP would let the flag admit a tensor-sharded rank. Assert in the aggregated test that the LLM still carries dwdp_config. create_py_executor either honours the config or raises, so this distinguishes a DWDP run from one where the config was dropped and MoE silently fell back to the normal parallel path -- which is also correct and would score the same. Keep the disaggregated tests in the QA list under their existing waive rather than dropping them entirely. The file imports helpers from test_disaggregated_serving.py and would rot unnoticed if nothing collected it, and keeping the waive makes dropping it the natural signal that the disaggregated path is healthy again. Only the pre-merge list entry is removed, so pre-merge no longer depends on disaggregated serving. Also note the world-size relationship the test relies on, and annotate the test parameters. Signed-off-by: tianyuz-nv --- .../_torch/pyexecutor/py_executor_creator.py | 34 +++++++++---------- .../defs/accuracy/test_dwdp_aggregated.py | 21 +++++++++++- .../test_dwdp_disaggregated_serving.py | 15 ++++---- .../test_lists/qa/llm_function_core.txt | 3 ++ tests/integration/test_lists/waives.txt | 3 ++ 5 files changed, 51 insertions(+), 25 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py b/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py index 352f02147755..f3f257f8465e 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor_creator.py @@ -538,34 +538,32 @@ def create_py_executor( # Initialize DWDP Manager. # - # DWDP requires every global MPI rank to be a complete, unsharded model - # replica owning one expert slice: `dwdp_rank = global_mpi_rank() % - # dwdp_size` is only a meaningful identity under that bijection, and the - # layout installed by `_init_dwdp_expert_layout` assumes expert weights are - # not additionally tensor-sharded. Two deployments satisfy this: - # * disaggregated serving -- each context worker is its own TP=1 instance; - # DWDP peers are separate instances joined by the global MPI world. - # * aggregated serving with attention DP -- one instance in which - # `mapping.dp_size == tp_size` (see Mapping.dp_size), so attention is - # replicated and data-parallel rather than tensor-sharded and each rank - # is still a full replica. Mapping already forces `moe_tp = moe_ep = 1` - # whenever `dwdp_size > 1`, so expert weights stay unsharded and - # ConfigurableMoE selects no MoE communication strategy. - # Real tensor parallelism (tp_size > 1 without attention DP) is rejected: a - # rank would be a shard of a replica, breaking both the rank-to-worker - # bijection and the unsharded-expert assumption. + # DWDP needs every global MPI rank to be a complete, unsharded model replica + # owning one expert slice -- `dwdp_rank = global_mpi_rank() % dwdp_size` is + # only meaningful under that bijection. Disaggregated context workers satisfy + # this with TP=1, aggregated serving with full attention DP; everything that + # shards a replica across ranks is rejected. Pipeline and context parallelism + # are rejected even under attention DP, since pairing ranks that hold + # different layers as DWDP peers gives wrong expert weights rather than an + # error. The TP check tests `dp_size == tp_size` rather than + # `enable_attention_dp` so a future partial attention DP cannot slip through. dwdp_manager: Optional[DwdpManager] = None if llm_args.dwdp_config is not None: if llm_args.dwdp_config.dwdp_size <= 1: raise ValueError( f"DWDP requires dwdp_size > 1, got {llm_args.dwdp_config.dwdp_size}." ) - if mapping.tp_size > 1 and not mapping.enable_attention_dp: + if mapping.pp_size > 1 or mapping.cp_size > 1: + raise ValueError( + "DWDP requires each rank to be a complete model replica, so " + "pipeline and context parallelism are not supported, but got " + f"pp_size={mapping.pp_size}, cp_size={mapping.cp_size}.") + if mapping.tp_size > 1 and mapping.dp_size != mapping.tp_size: raise ValueError( "DWDP requires each rank to be a complete model replica: use " "tp_size=1 (disaggregated context worker) or " "enable_attention_dp=True (aggregated serving), but got " - f"tp_size={mapping.tp_size} with enable_attention_dp=False.") + f"tp_size={mapping.tp_size}, dp_size={mapping.dp_size}.") dwdp_manager = DwdpManager(config=llm_args.dwdp_config, dist=dist, mapping=mapping) diff --git a/tests/integration/defs/accuracy/test_dwdp_aggregated.py b/tests/integration/defs/accuracy/test_dwdp_aggregated.py index c3448f43406f..0c5623096111 100644 --- a/tests/integration/defs/accuracy/test_dwdp_aggregated.py +++ b/tests/integration/defs/accuracy/test_dwdp_aggregated.py @@ -45,6 +45,11 @@ # Three remote peers per rank instead of one is also what makes # ``contention_opt`` meaningful, since it interleaves prefetch slices across # peers to spread them over several NVLink links. +# +# The MPI world must be exactly ``num_groups * dwdp_size`` ranks -- a rank +# computes ``group_id = rank // dwdp_size`` and DwdpManager rejects +# ``group_id >= num_groups``. ``tensor_parallel_size`` below is that world size, +# so it has to track DWDP_SIZE and the ``num_groups=1`` passed to DwdpConfig. DWDP_SIZE = 4 @@ -63,7 +68,12 @@ class TestDwdpAggDeepSeekV3Lite(LlmapiAccuracyTestHarness): ], ids=["mode_a_uniform", "mode_b_overlap", "mode_a_uniform_contention_opt"], ) - def test_dwdp_agg_accuracy(self, num_experts_per_worker, num_prefetch_experts, contention_opt): + def test_dwdp_agg_accuracy( + self, + num_experts_per_worker: int, + num_prefetch_experts: int, + contention_opt: bool, + ) -> None: dwdp_config = DwdpConfig( dwdp_size=DWDP_SIZE, num_groups=1, @@ -94,5 +104,14 @@ def test_dwdp_agg_accuracy(self, num_experts_per_worker, num_prefetch_experts, c tokens_per_block=32, ), ) as llm: + # Guard against DWDP silently not running at all. If the config were + # dropped before create_py_executor, MoE would fall back to the normal + # parallel path, which is also correct and scores the same, so the + # accuracy check below could not tell the two apart. create_py_executor + # either honours dwdp_config or raises -- it has no branch that ignores + # it -- so an LLM that constructed successfully while still carrying the + # config had a DwdpManager built for it. + assert llm.args.dwdp_config == dwdp_config + task = GSM8K(self.MODEL_NAME) task.evaluate(llm) diff --git a/tests/integration/defs/accuracy/test_dwdp_disaggregated_serving.py b/tests/integration/defs/accuracy/test_dwdp_disaggregated_serving.py index 386f9891e193..5b9ed7e2b215 100644 --- a/tests/integration/defs/accuracy/test_dwdp_disaggregated_serving.py +++ b/tests/integration/defs/accuracy/test_dwdp_disaggregated_serving.py @@ -3,12 +3,15 @@ Separated from test_disaggregated_serving.py to isolate MPI-dependent test infrastructure for easier maintenance. -NOTE: these tests are intentionally not registered in any CI test list. DWDP -accuracy is gated in CI by test_dwdp_aggregated.py, which exercises the same -expert-sharing paths without the disaggregated KV cache transceiver and is -therefore not exposed to per-cluster transport configuration. This file is kept -as a manual reproduction of the disaggregated DWDP path; run it directly with -pytest. +NOTE: these tests no longer gate pre-merge CI. DWDP accuracy is gated by +test_dwdp_aggregated.py, which exercises the same expert-sharing paths without +the disaggregated KV cache transceiver and is therefore not exposed to +per-cluster transport configuration. The tests here stay registered in the QA +list, waived under nvbugs/6276923, so that the file keeps being collected -- +it imports helpers from test_disaggregated_serving.py and would otherwise rot +unnoticed -- and so that dropping the waive is the natural signal once the +disaggregated path is healthy again. They also remain useful as a manual +reproduction of the disaggregated DWDP path. When running it manually, check the launcher's UCX settings first: SLURM enroot/pyxis injects ``UCX_TLS=tcp`` from the host MPI stack on some clusters, diff --git a/tests/integration/test_lists/qa/llm_function_core.txt b/tests/integration/test_lists/qa/llm_function_core.txt index 5919c8dbdb37..758a360d0ef4 100644 --- a/tests/integration/test_lists/qa/llm_function_core.txt +++ b/tests/integration/test_lists/qa/llm_function_core.txt @@ -70,6 +70,9 @@ accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_nixl_backend accuracy/test_dwdp_aggregated.py::TestDwdpAggDeepSeekV3Lite::test_dwdp_agg_accuracy[mode_a_uniform] accuracy/test_dwdp_aggregated.py::TestDwdpAggDeepSeekV3Lite::test_dwdp_agg_accuracy[mode_b_overlap] accuracy/test_dwdp_aggregated.py::TestDwdpAggDeepSeekV3Lite::test_dwdp_agg_accuracy[mode_a_uniform_contention_opt] +accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy +accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy_contention_opt +accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy_mode_b_overlap accuracy/test_epd_disagg_multimodal.py::TestVideoMMEEPD::test_disaggregated_videomme[nemotron_nano_v3_omni_fp8] accuracy/test_epd_disagg_multimodal.py::TestVideoMMEEPD::test_disaggregated_videomme[nemotron_nano_v3_omni_nvfp4] accuracy/test_epd_disagg_multimodal.py::TestVideoMMEEPD::test_disaggregated_videomme[qwen3vl_2b_instruct] diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index c478493daa26..abc96646e567 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -181,6 +181,9 @@ full:DGX_H100/unittest/llmapi/test_llm_multi_gpu_pytorch.py -m "gpu4" SKIP (http full:DGX_H100/unittest/llmapi/test_llm_multi_gpu_pytorch.py::test_llm_get_stats_pp4[False-False-True] SKIP (https://nvbugs/6618098) full:DGX_H100/unittest/llmapi/test_llm_multi_gpu_pytorch.py::test_tinyllama_logits_processor_tp2pp2 SKIP (https://nvbugs/6618106) full:GB200/accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[ctx_block_reuse_only] SKIP (https://nvbugs/6525893) +full:GB200/accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy SKIP (https://nvbugs/6276923) +full:GB200/accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy_contention_opt SKIP (https://nvbugs/6276923) +full:GB200/accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy_mode_b_overlap SKIP (https://nvbugs/6276923) full:GB200/accuracy/test_llm_api_autodeploy.py::TestNemotronNanoV3::test_accuracy[fp8-4-attn_dp_off-trtllm] SKIP (https://nvbugs/6539942) full:GB200/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=False] SKIP (https://nvbugs/6525896) full:GB200/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_bf16_gemm[cuda_graph=True] SKIP (https://nvbugs/6525897) From e8cfdcdca7f7078b5ae00adf087faaaadc84f65c Mon Sep 17 00:00:00 2001 From: tianyuz-nv Date: Thu, 13 Aug 2026 19:31:37 -0700 Subject: [PATCH 3/3] [None][fix] Size MoE chunks by local rows when DWDP has no dispatch `ConfigurableMoE.calculate_num_chunks` only recognised two shapes: DP with a comm strategy (rows = num_dp_ranks * max_tokens_per_rank after dispatch) and non-DP, which asserts the caller passed a single-element `all_rank_num_tokens`. DWDP is a third shape. It prefetches expert weights to every rank instead of dispatching tokens to experts, so `_create_comm_strategy_auto` returns None and the non-DP branch is taken. Disaggregated context workers pass its assert only because they run tp_size=1, so the list has one entry anyway. Aggregated serving with attention DP passes one entry per DP rank, the assert fires, and the executor worker dies during attention warmup: non-DP path expects a single-element list, got 4 Size the chunks from `max(all_rank_num_tokens)` when DWDP is on: without a dispatch a rank only ever processes its own tokens, never more. The branch is keyed off `enable_dwdp` rather than `comm is None` so that no non-DWDP configuration changes the branch it takes. `enable_dwdp` is assigned once in `__init__`, before both the comm strategy and the scheduler are built, and implies `comm is None`, so the new branch is always reached when DWDP is on and never reachable otherwise. Single-element lists are unaffected either way, since `max([n]) == n`. Signed-off-by: tianyuz-nv --- tensorrt_llm/_torch/modules/fused_moe/configurable_moe.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tensorrt_llm/_torch/modules/fused_moe/configurable_moe.py b/tensorrt_llm/_torch/modules/fused_moe/configurable_moe.py index 72e504b48f45..31a5cb88477b 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/configurable_moe.py +++ b/tensorrt_llm/_torch/modules/fused_moe/configurable_moe.py @@ -487,6 +487,11 @@ def calculate_num_chunks(self, all_rank_num_tokens: List[int]) -> int: """ if self.use_dp and self.comm is not None: num_rows = self._dp_padded_num_rows(all_rank_num_tokens) + elif self.enable_dwdp: + # DWDP prefetches expert weights instead of dispatching tokens, so a + # rank only ever processes its own tokens, never more. Keyed off + # ``enable_dwdp`` so no non-DWDP path changes the branch it takes. + num_rows = max(all_rank_num_tokens) else: # non-DP: no cross-rank dispatch. The scheduler fills all_rank_num_tokens # from [x.shape[0]] before calling here, so it must be a single-element list.