From a27aeec614f392605b8e440d89ddbfc4712630eb Mon Sep 17 00:00:00 2001 From: Shixiaowei02 <39303645+Shixiaowei02@users.noreply.github.com> Date: Mon, 20 Jul 2026 09:15:22 +0000 Subject: [PATCH 1/7] [None][test] trim disagg Llama test_auto_dtype from 16 combos to 2 The 2**4 cross-product of ctx/gen overlap-scheduler x ctx/gen block-reuse ran a full MMLU+GSM8K eval per combo. Overlap scheduler is token-invariant (covered by unit tests test_benchmark_disagg.py and test_disagg_gen_transfer_gate.py), so only the block-reuse axis changes which KV blocks are transferred. Keep block-reuse on/off with overlap enabled (production default) and drop the other 14 combos from CI (l0_dgx_h100, llm_function_core). Signed-off-by: Shixiaowei02 <39303645+Shixiaowei02@users.noreply.github.com> --- .../accuracy/test_disaggregated_serving.py | 16 +++++++--------- .../test_lists/qa/llm_function_core.txt | 18 ++---------------- .../test_lists/test-db/l0_dgx_h100.yml | 18 ++---------------- 3 files changed, 11 insertions(+), 41 deletions(-) diff --git a/tests/integration/defs/accuracy/test_disaggregated_serving.py b/tests/integration/defs/accuracy/test_disaggregated_serving.py index 0a970516d838..4091b4b9cc94 100644 --- a/tests/integration/defs/accuracy/test_disaggregated_serving.py +++ b/tests/integration/defs/accuracy/test_disaggregated_serving.py @@ -623,15 +623,13 @@ class TestLlama3_1_8BInstruct(LlmapiAccuracyTestHarness): @skip_pre_hopper @pytest.mark.skip_less_device(2) - @pytest.mark.parametrize("ctx_disable_overlap_scheduler", [False, True]) - @pytest.mark.parametrize("gen_disable_overlap_scheduler", [False, True]) - @pytest.mark.parametrize("ctx_enable_block_reuse", [True, False]) - @pytest.mark.parametrize("gen_enable_block_reuse", [True, False]) - def test_auto_dtype(self, ctx_disable_overlap_scheduler, - gen_disable_overlap_scheduler, ctx_enable_block_reuse, - gen_enable_block_reuse): + # overlap scheduler is token-invariant (unit-tested); only block-reuse changes which KV is transferred + @pytest.mark.parametrize("ctx_enable_block_reuse,gen_enable_block_reuse", + [(True, True), (False, False)], + ids=["block_reuse", "no_block_reuse"]) + def test_auto_dtype(self, ctx_enable_block_reuse, gen_enable_block_reuse): ctx_server_config = { - "disable_overlap_scheduler": ctx_disable_overlap_scheduler, + "disable_overlap_scheduler": False, "kv_cache_config": { "enable_block_reuse": ctx_enable_block_reuse } @@ -641,7 +639,7 @@ def test_auto_dtype(self, ctx_disable_overlap_scheduler, "max_tokens_in_buffer": 4096 } gen_server_config = { - "disable_overlap_scheduler": gen_disable_overlap_scheduler, + "disable_overlap_scheduler": False, "kv_cache_config": { "enable_block_reuse": gen_enable_block_reuse } diff --git a/tests/integration/test_lists/qa/llm_function_core.txt b/tests/integration/test_lists/qa/llm_function_core.txt index 9ed9c317c98e..28543e372aec 100644 --- a/tests/integration/test_lists/qa/llm_function_core.txt +++ b/tests/integration/test_lists/qa/llm_function_core.txt @@ -21,22 +21,8 @@ accuracy/test_disaggregated_serving.py::TestGPTOSS::test_kv_cache_v2_nixl_python accuracy/test_disaggregated_serving.py::TestGPTOSS::test_kv_cache_v2_nixl_python[cache_mgr_v2] accuracy/test_disaggregated_serving.py::TestGLM52NVFP4::test_nvfp4_nixl[cache_mgr_v1] accuracy/test_disaggregated_serving.py::TestKimiK25::test_nvfp4 -accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-False-False-False] -accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-False-False-True] -accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-False-True-False] -accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-False-True-True] -accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-True-False-False] -accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-True-False-True] -accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-True-True-False] -accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-True-True-True] -accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-False-False-False] -accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-False-False-True] -accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-False-True-False] -accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-False-True-True] -accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-True-False-False] -accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-True-False-True] -accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-True-True-False] -accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-True-True-True] +accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[block_reuse] +accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[no_block_reuse] accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=1-ctx_pp=2] accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=1-ctx_pp=4] accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=2-ctx_pp=2] diff --git a/tests/integration/test_lists/test-db/l0_dgx_h100.yml b/tests/integration/test_lists/test-db/l0_dgx_h100.yml index 395038927cf0..881c0c5615ed 100644 --- a/tests/integration/test_lists/test-db/l0_dgx_h100.yml +++ b/tests/integration/test_lists/test-db/l0_dgx_h100.yml @@ -42,22 +42,8 @@ l0_dgx_h100: - accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_auto_dtype[True] - accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_kv_cache_v2_nixl_python[cache_mgr_v1] - accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_kv_cache_v2_nixl_python[cache_mgr_v2] - - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-False-False-False] - - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-False-False-True] - - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-False-True-False] - - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-False-True-True] - - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-True-False-False] - - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-True-False-True] - - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-True-True-False] - - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-True-True-True] - - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-False-False-False] - - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-False-False-True] - - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-False-True-False] - - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-False-True-True] - - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-True-False-False] - - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-True-False-True] - - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-True-True-False] - - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[True-True-True-True] + - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[block_reuse] + - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[no_block_reuse] - unittest/llmapi/apps/test_disagg_serving_perf_metrics.py - disaggregated/test_disaggregated.py::test_disaggregated_cancel_large_context_requests[DeepSeek-V3-Lite-bf16] # llmapi From 7080d692d1dcf6f0c6a199f8a577548457e24e23 Mon Sep 17 00:00:00 2001 From: Shixiaowei02 <39303645+Shixiaowei02@users.noreply.github.com> Date: Mon, 20 Jul 2026 09:23:21 +0000 Subject: [PATCH 2/7] [None][test] drop redundant llguidance backend axis from disagg guided_decoding Guided decoding runs entirely on the generation worker and the KV-cache handoff is grammar-backend-agnostic, so exercising both xgrammar and llguidance under disaggregated serving only re-validates grammar-engine correctness that the aggregated guided-decoding tests already cover (test_llm_api_pytorch.py has 7 backend-parametrized sites). Keep xgrammar (the default) for the disagg plumbing guard and the DeepSeek guided+MTP interaction; drop the llguidance combos from TestLlama3_1_8BInstruct/TestDeepSeekV3Lite::test_guided_decoding (7 CI entries across l0_rtx_pro_6000, llm_function_core, llm_function_rtx6k). Signed-off-by: Shixiaowei02 <39303645+Shixiaowei02@users.noreply.github.com> --- .../integration/defs/accuracy/test_disaggregated_serving.py | 6 ++++-- tests/integration/test_lists/qa/llm_function_core.txt | 3 --- tests/integration/test_lists/qa/llm_function_rtx6k.txt | 2 -- tests/integration/test_lists/test-db/l0_rtx_pro_6000.yml | 2 -- 4 files changed, 4 insertions(+), 9 deletions(-) diff --git a/tests/integration/defs/accuracy/test_disaggregated_serving.py b/tests/integration/defs/accuracy/test_disaggregated_serving.py index 4091b4b9cc94..6b7270c1d387 100644 --- a/tests/integration/defs/accuracy/test_disaggregated_serving.py +++ b/tests/integration/defs/accuracy/test_disaggregated_serving.py @@ -918,7 +918,8 @@ def test_gen_only_spec_dec(self): @pytest.mark.skip_less_device(2) @pytest.mark.skip_less_device_memory(32000) - @pytest.mark.parametrize("backend", ["xgrammar", "llguidance"]) + # grammar backend is disagg-agnostic (runs on gen worker); backend correctness is covered by aggregated tests + @pytest.mark.parametrize("backend", ["xgrammar"]) def test_guided_decoding(self, backend: str, mocker): mocker.patch.dict(os.environ, {"TRTLLM_XGUIDANCE_LENIENT": "1"}) ctx_server_config = { @@ -1272,7 +1273,8 @@ def test_auto_dtype(self, overlap_scheduler, mtp_nextn): @pytest.mark.skip_less_device(2) @pytest.mark.skip_less_device_memory(60000) @parametrize_with_ids("mtp_nextn", [0, 2]) - @pytest.mark.parametrize("backend", ["xgrammar", "llguidance"]) + # grammar backend is disagg-agnostic (runs on gen worker); backend correctness is covered by aggregated tests + @pytest.mark.parametrize("backend", ["xgrammar"]) def test_guided_decoding(self, backend: str, mtp_nextn: int, mocker): mocker.patch.dict(os.environ, {"TRTLLM_XGUIDANCE_LENIENT": "1"}) ctx_server_config = { diff --git a/tests/integration/test_lists/qa/llm_function_core.txt b/tests/integration/test_lists/qa/llm_function_core.txt index 28543e372aec..8b4710c5adee 100644 --- a/tests/integration/test_lists/qa/llm_function_core.txt +++ b/tests/integration/test_lists/qa/llm_function_core.txt @@ -7,8 +7,6 @@ accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_ accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype_with_helix[fifo_v2-cudagraph:with_padding-pp1dp2cp2] accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype_with_helix[fifo_v2-cudagraph:with_padding-pp1tp2cp2] accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_gen_only_sync -accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=0] -accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=2] accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=0] accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=2] accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_auto_dtype[False] @@ -33,7 +31,6 @@ accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[MMLU-gen_tp=2-ctx_pp=4] accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_eagle3[eagle3_one_model=False-overlap_scheduler=False] accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_eagle3[eagle3_one_model=True-overlap_scheduler=True] -accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding[llguidance] accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding[xgrammar] accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[llguidance-eagle3_one_model=False] accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[llguidance-eagle3_one_model=True] diff --git a/tests/integration/test_lists/qa/llm_function_rtx6k.txt b/tests/integration/test_lists/qa/llm_function_rtx6k.txt index a354d3a9a1a6..0ad2b58d7e21 100644 --- a/tests/integration/test_lists/qa/llm_function_rtx6k.txt +++ b/tests/integration/test_lists/qa/llm_function_rtx6k.txt @@ -2,8 +2,6 @@ accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_ accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=0-overlap_scheduler=True] accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=False] accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=True] -accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=0] -accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=2] accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=0] accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=2] accuracy/test_disaggregated_serving.py::TestGPTOSS::test_auto_dtype[False] diff --git a/tests/integration/test_lists/test-db/l0_rtx_pro_6000.yml b/tests/integration/test_lists/test-db/l0_rtx_pro_6000.yml index 05e658f190e5..c23ec039fef5 100644 --- a/tests/integration/test_lists/test-db/l0_rtx_pro_6000.yml +++ b/tests/integration/test_lists/test-db/l0_rtx_pro_6000.yml @@ -115,8 +115,6 @@ l0_rtx_pro_6000: # - accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_auto_dtype[True] - accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=0] - accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[xgrammar-mtp_nextn=2] - - accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=0] - - accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=2] - accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4_4gpus[latency_moe_cutlass] # 20 mins # - accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_nvfp4_4gpus[latency_moe_cutlass_eagle3] # failed - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[dep4_latency_moe_cutlass-torch_compile=True] From d25f94814a6d58c102df2329664ca2f4e6177ebd Mon Sep 17 00:00:00 2001 From: Shixiaowei02 <39303645+Shixiaowei02@users.noreply.github.com> Date: Mon, 20 Jul 2026 09:57:57 +0000 Subject: [PATCH 3/7] [None][test] remove UCX-specialized disagg tests and redundant Llama kv_cache_v2 UCX cache-transceiver coverage is model-agnostic plumbing already exercised by the cheap unit/C++ tests (unittest/disaggregated UCX params, cpp/test_multi_gpu test_cache_transceiver ucx_kvcache), so the expensive e2e UCX-specialized integration tests are redundant. Remove test_disaggregated_deepseek_v3_lite_fp8_ucx and test_disaggregated_deepseek_v3_lite_fp8_ucx_tp1_single_gpu, plus their now orphaned config disagg_config_ctxtp2_gentp2_deepseek_v3_lite_ucx.yaml and its config-map entry. Also remove TestLlama3_1_8BInstruct::test_kv_cache_v2_nixl_python: the KVCacheManager-v2 + NIXL + PYTHON transceiver path is model-agnostic and byte-covered by test_cache_transceiver_single_process.py; TestDeepSeekV3Lite keeps the canonical MLA representative. Signed-off-by: Shixiaowei02 <39303645+Shixiaowei02@users.noreply.github.com> --- .../accuracy/test_disaggregated_serving.py | 44 ------------------- ...ig_ctxtp2_gentp2_deepseek_v3_lite_ucx.yaml | 17 ------- .../defs/disaggregated/test_disaggregated.py | 42 ------------------ .../test_lists/qa/llm_function_core.txt | 3 -- .../test_lists/test-db/l0_dgx_b200.yml | 1 - .../test_lists/test-db/l0_dgx_b300.yml | 2 - .../test_lists/test-db/l0_dgx_h100.yml | 2 - .../test_lists/test-db/l0_h100.yml | 1 - 8 files changed, 112 deletions(-) delete mode 100644 tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_deepseek_v3_lite_ucx.yaml diff --git a/tests/integration/defs/accuracy/test_disaggregated_serving.py b/tests/integration/defs/accuracy/test_disaggregated_serving.py index 6b7270c1d387..ff80f706306c 100644 --- a/tests/integration/defs/accuracy/test_disaggregated_serving.py +++ b/tests/integration/defs/accuracy/test_disaggregated_serving.py @@ -711,50 +711,6 @@ def test_beam_search(self): extra_acc_spec=f"beam_width={max_beam_width}", sampling_params=sampling_params) - @skip_pre_hopper - @pytest.mark.skip_less_device(2) - def test_kv_cache_v2_nixl_python(self): - """Test with use_kv_cache_manager_v2=True, block_reuse=False, backend=NIXL, transceiver_runtime=PYTHON.""" - ctx_server_config = { - "disable_overlap_scheduler": True, - "kv_cache_config": { - "enable_block_reuse": False, - "use_kv_cache_manager_v2": True - }, - "cache_transceiver_config": { - "backend": "NIXL", - "transceiver_runtime": "PYTHON" - } - } - gen_server_config = { - "disable_overlap_scheduler": False, - "kv_cache_config": { - "enable_block_reuse": False, - "use_kv_cache_manager_v2": True - }, - "cache_transceiver_config": { - "backend": "NIXL", - "transceiver_runtime": "PYTHON" - } - } - disaggregated_server_config = { - "hostname": "localhost", - "port": 8000, - "backend": "pytorch", - "context_servers": { - "num_instances": 1, - "urls": ["localhost:8001"] - }, - "generation_servers": { - "num_instances": 1, - "urls": ["localhost:8002"] - } - } - with launch_disaggregated_llm(disaggregated_server_config, - ctx_server_config, gen_server_config, - self.MODEL_PATH) as llm: - run_accuracy_test(llm, self.MODEL_NAME, ["GSM8K"]) - @pytest.mark.skip_less_device(2) def test_ngram(self): speculative_decoding_config = { diff --git a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_deepseek_v3_lite_ucx.yaml b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_deepseek_v3_lite_ucx.yaml deleted file mode 100644 index 6ab1ba3e6174..000000000000 --- a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_deepseek_v3_lite_ucx.yaml +++ /dev/null @@ -1,17 +0,0 @@ -hostname: localhost -model: DeepSeek-V3-Lite/fp8 -free_gpu_memory_fraction: 0.25 -backend: pytorch -disable_overlap_scheduler: true -context_servers: - num_instances: 1 - tensor_parallel_size: 2 - pipeline_parallel_size: 1 - cache_transceiver_config: - backend: UCX -generation_servers: - num_instances: 1 - tensor_parallel_size: 2 - pipeline_parallel_size: 1 - cache_transceiver_config: - backend: UCX diff --git a/tests/integration/defs/disaggregated/test_disaggregated.py b/tests/integration/defs/disaggregated/test_disaggregated.py index 1234dbe51504..eeb4c37f9305 100644 --- a/tests/integration/defs/disaggregated/test_disaggregated.py +++ b/tests/integration/defs/disaggregated/test_disaggregated.py @@ -346,10 +346,6 @@ def get_test_config(test_desc, example_dir, test_root): f"{test_configs_root}/disagg_config_ctxpp4_gentp4.yaml", "deepseek_v3_lite_fp8_mpi": f"{test_configs_root}/disagg_config_ctxtp2_gentp2_deepseek_v3_lite_mpi.yaml", - "deepseek_v3_lite_fp8_tp1_ucx": - f"{test_configs_root}/disagg_config_ctxtp1_gentp1_deepseek_v3_lite_ucx.yaml", - "deepseek_v3_lite_fp8_tp2_ucx": - f"{test_configs_root}/disagg_config_ctxtp2_gentp2_deepseek_v3_lite_ucx.yaml", "deepseek_v3_lite_fp8_nixl": f"{test_configs_root}/disagg_config_ctxtp2_gentp2_deepseek_v3_lite_nixl.yaml", "deepseek_v3_lite_fp8_tp1": @@ -2020,26 +2016,6 @@ def test_disaggregated_deepseek_v3_lite_fp8_ctxtp2ep2pp2_gentp4_one_mtp_block_re @skip_no_hopper @skip_arm -@pytest.mark.skip_less_device(4) -@pytest.mark.parametrize("deepseek_v3_model_root", ['DeepSeek-V3-Lite-fp8'], - indirect=True) -def test_disaggregated_deepseek_v3_lite_fp8_ucx(disaggregated_test_root, - disaggregated_example_root, - llm_venv, - deepseek_v3_model_root): - - setup_model_symlink(llm_venv, deepseek_v3_model_root, - "DeepSeek-V3-Lite/fp8") - env = llm_venv._new_env.copy() - env["TRTLLM_USE_UCX_KVCACHE"] = "1" - env["UCX_TLS"] = get_ucx_tls() - run_disaggregated_test(disaggregated_example_root, - "deepseek_v3_lite_fp8_tp2_ucx", - env=env, - model_path=deepseek_v3_model_root, - cwd=llm_venv.get_working_directory()) - - @skip_no_hopper @skip_arm @pytest.mark.parametrize("deepseek_v3_model_root", ['DeepSeek-V3-Lite-fp8'], @@ -2062,24 +2038,6 @@ def test_disaggregated_deepseek_v3_lite_fp8_nixl(disaggregated_test_root, cwd=llm_venv.get_working_directory()) -@skip_no_hopper -@skip_arm -@pytest.mark.parametrize("deepseek_v3_model_root", ['DeepSeek-V3-Lite-fp8'], - indirect=True) -def test_disaggregated_deepseek_v3_lite_fp8_ucx_tp1_single_gpu( - disaggregated_test_root, disaggregated_example_root, llm_venv, - deepseek_v3_model_root): - setup_model_symlink(llm_venv, deepseek_v3_model_root, - "DeepSeek-V3-Lite/fp8") - env = llm_venv._new_env.copy() - env["TRTLLM_USE_UCX_KVCACHE"] = "1" - env["UCX_TLS"] = get_ucx_tls() - - run_disaggregated_test(disaggregated_example_root, - "deepseek_v3_lite_fp8_tp1_ucx", - env=env, - model_path=deepseek_v3_model_root, - cwd=llm_venv.get_working_directory()) @skip_no_hopper diff --git a/tests/integration/test_lists/qa/llm_function_core.txt b/tests/integration/test_lists/qa/llm_function_core.txt index 8b4710c5adee..b3b6a31d33cb 100644 --- a/tests/integration/test_lists/qa/llm_function_core.txt +++ b/tests/integration/test_lists/qa/llm_function_core.txt @@ -36,7 +36,6 @@ accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_dec accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[llguidance-eagle3_one_model=True] accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[xgrammar-eagle3_one_model=False] accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_guided_decoding_with_eagle3[xgrammar-eagle3_one_model=True] -accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_kv_cache_v2_nixl_python accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_multi_instance[GSM8K] accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_multi_instance[MMLU] accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ngram @@ -860,8 +859,6 @@ disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1 disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1_single_gpu[DeepSeek-V3-Lite-fp8] disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1_single_gpu_mtp[DeepSeek-V3-Lite-fp8] disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1_two_mtp[DeepSeek-V3-Lite-fp8] -disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_ucx[DeepSeek-V3-Lite-fp8] -disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_ucx_tp1_single_gpu[DeepSeek-V3-Lite-fp8] disaggregated/test_disaggregated.py::test_disaggregated_diff_max_tokens[TinyLlama-1.1B-Chat-v1.0] disaggregated/test_disaggregated.py::test_disaggregated_genbs1[TinyLlama-1.1B-Chat-v1.0] disaggregated/test_disaggregated.py::test_disaggregated_gpt_oss_120b_harmony[gpt_oss/gpt-oss-120b] diff --git a/tests/integration/test_lists/test-db/l0_dgx_b200.yml b/tests/integration/test_lists/test-db/l0_dgx_b200.yml index 1fe5ad179f58..243738f3d405 100644 --- a/tests/integration/test_lists/test-db/l0_dgx_b200.yml +++ b/tests/integration/test_lists/test-db/l0_dgx_b200.yml @@ -46,7 +46,6 @@ l0_dgx_b200: - accuracy/test_llm_api_pytorch.py::TestQwen3_5_397B_A17B::test_nvfp4_4gpus_static_eplb[moe_backend=CUTEDSL] - accuracy/test_llm_api_pytorch.py::TestQwen3_5_397B_A17B::test_nvfp4_4gpus_static_eplb[moe_backend=TRTLLM] - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[tep4_latency_moe_trtllm-torch_compile=True] - - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_ucx[DeepSeek-V3-Lite-fp8] - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_nixl[DeepSeek-V3-Lite-fp8] - disaggregated/test_disaggregated.py::test_disaggregated_gpt_oss_120b_harmony[gpt_oss/gpt-oss-120b] - accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[latency_adp_lmtp_tp4] diff --git a/tests/integration/test_lists/test-db/l0_dgx_b300.yml b/tests/integration/test_lists/test-db/l0_dgx_b300.yml index 5394ee8bc016..417c989d9f41 100644 --- a/tests/integration/test_lists/test-db/l0_dgx_b300.yml +++ b/tests/integration/test_lists/test-db/l0_dgx_b300.yml @@ -60,7 +60,6 @@ l0_dgx_b300: - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[dep4_latency_moe_trtllm-torch_compile=False] - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[dep4_latency_moe_cutlass-torch_compile=True] - accuracy/test_disaggregated_serving.py::TestQwen3_30B_A3B::test_mixed_ctx_gen_model[ctxpp2gentp2] - - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_ucx[DeepSeek-V3-Lite-fp8] - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-tp4-cutlass-fp8] - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v1_kv_cache-tp4-cutlass-auto] - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-tp4-cutlass-auto] @@ -68,7 +67,6 @@ l0_dgx_b300: - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4a16[dp4-fp8] - accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_nixl_backend - accuracy/test_disaggregated_serving.py::TestDeepSeekR1::test_kv_cache_v2_nixl_python[cache_mgr_v1] - - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_kv_cache_v2_nixl_python - accuracy/test_disaggregated_serving.py::TestGPTOSS::test_kv_cache_v2_nixl_python[cache_mgr_v1] - accuracy/test_disaggregated_serving.py::TestGPTOSS::test_kv_cache_v2_nixl_python[cache_mgr_v2] - accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[throughput_tp4] TIMEOUT (180) diff --git a/tests/integration/test_lists/test-db/l0_dgx_h100.yml b/tests/integration/test_lists/test-db/l0_dgx_h100.yml index 881c0c5615ed..ffefc4a3574f 100644 --- a/tests/integration/test_lists/test-db/l0_dgx_h100.yml +++ b/tests/integration/test_lists/test-db/l0_dgx_h100.yml @@ -37,7 +37,6 @@ l0_dgx_h100: - accuracy/test_disaggregated_serving.py::TestQwen3_8B::test_nixl_backend - accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_gen_only_sync - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ngram - - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_kv_cache_v2_nixl_python - accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_auto_dtype[False] - accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_auto_dtype[True] - accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_kv_cache_v2_nixl_python[cache_mgr_v1] @@ -201,7 +200,6 @@ l0_dgx_h100: - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_fp8_block_scales_cuda_graph_padding_4gpus[attention_dp=True-mtp_nextn=2] - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding_4gpus[xgrammar-mtp_nextn=2] - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_mpi[DeepSeek-V3-Lite-fp8] - - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_ucx[DeepSeek-V3-Lite-fp8] - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_nixl[DeepSeek-V3-Lite-fp8] - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_attention_dp[DeepSeek-V3-Lite-fp8] - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_attention_dp_overlap[DeepSeek-V3-Lite-fp8] diff --git a/tests/integration/test_lists/test-db/l0_h100.yml b/tests/integration/test_lists/test-db/l0_h100.yml index 88130605c153..d1d1081477e6 100644 --- a/tests/integration/test_lists/test-db/l0_h100.yml +++ b/tests/integration/test_lists/test-db/l0_h100.yml @@ -172,7 +172,6 @@ l0_h100: - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1_single_gpu[DeepSeek-V3-Lite-fp8] - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1_single_gpu_mtp[DeepSeek-V3-Lite-fp8] - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_tp1_two_mtp[DeepSeek-V3-Lite-fp8] - - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_ucx_tp1_single_gpu[DeepSeek-V3-Lite-fp8] - disaggregated/test_disaggregated.py::test_disaggregated_load_balance[TinyLlama-1.1B-Chat-v1.0] - disaggregated/test_disaggregated.py::test_disaggregated_tinyllama_multi_orchestrator[TinyLlama-1.1B-Chat-v1.0] - disaggregated/test_disaggregated_single_gpu.py::test_disaggregated_simple_deepseek[False-False-DeepSeek-V3-Lite-fp8/fp8] From 54fede140922ec6071d8660baf11d3f812f09486 Mon Sep 17 00:00:00 2001 From: Shixiaowei02 <39303645+Shixiaowei02@users.noreply.github.com> Date: Mon, 20 Jul 2026 10:46:18 +0000 Subject: [PATCH 4/7] [None][test] de-UCX-specialize disagg helix/chunked tests; drop UCX logprobs test Switch the hardcoded backend=UCX in TestDeepSeekV3Lite/TestQwen3_8B test_auto_dtype_with_helix and the TestQwen3_8B chunked-prefill helper to DEFAULT, matching the rest of the accuracy suite (these tests cover helix and chunked prefill, not UCX specifically). Remove test_disaggregated_logprobs_serving and the llama31_8b_ucx config-map entry plus the orphaned yaml it exclusively used. Signed-off-by: Shixiaowei02 <39303645+Shixiaowei02@users.noreply.github.com> --- .../accuracy/test_disaggregated_serving.py | 10 +- ...g_config_ctxtp2_gentp2_llama31_8b_ucx.yaml | 18 -- .../defs/disaggregated/test_disaggregated.py | 203 ------------------ .../test_lists/qa/llm_function_core.txt | 1 - tests/integration/test_lists/waives.txt | 1 - 5 files changed, 5 insertions(+), 228 deletions(-) delete mode 100644 tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_llama31_8b_ucx.yaml diff --git a/tests/integration/defs/accuracy/test_disaggregated_serving.py b/tests/integration/defs/accuracy/test_disaggregated_serving.py index ff80f706306c..fb3d98d8b072 100644 --- a/tests/integration/defs/accuracy/test_disaggregated_serving.py +++ b/tests/integration/defs/accuracy/test_disaggregated_serving.py @@ -1143,7 +1143,7 @@ def test_auto_dtype_with_helix(self, comms_medium, cuda_graph_config, "enable_chunked_prefill": False, "cuda_graph_config": None, "cache_transceiver_config": { - "backend": "UCX", + "backend": "DEFAULT", "max_tokens_in_buffer": 8192, }, } @@ -1163,7 +1163,7 @@ def test_auto_dtype_with_helix(self, comms_medium, cuda_graph_config, "enable_chunked_prefill": False, "cuda_graph_config": cuda_graph_config, "cache_transceiver_config": { - "backend": "UCX", + "backend": "DEFAULT", "max_tokens_in_buffer": 8192, }, "enable_attention_dp": enable_attention_dp, @@ -1737,7 +1737,7 @@ def _test_chunked_prefill_helper(self, *, ctx_pp: int): "disable_overlap_scheduler": True, "cuda_graph_config": None, "cache_transceiver_config": { - "backend": "UCX", + "backend": "DEFAULT", "max_tokens_in_buffer": 4096 }, "enable_chunked_prefill": True, @@ -1748,7 +1748,7 @@ def _test_chunked_prefill_helper(self, *, ctx_pp: int): gen_server_config = { "cuda_graph_config": None, "cache_transceiver_config": { - "backend": "UCX", + "backend": "DEFAULT", "max_tokens_in_buffer": 4096 }, "max_batch_size": max_batch_size, @@ -1809,7 +1809,7 @@ def test_auto_dtype_with_helix(self, comms_medium, cuda_graph_config, "tokens_per_block": 32, } cache_transceiver_config = { - "backend": "UCX", + "backend": "DEFAULT", "max_tokens_in_buffer": 8192, } ctx_server_config = { diff --git a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_llama31_8b_ucx.yaml b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_llama31_8b_ucx.yaml deleted file mode 100644 index 243654b291a2..000000000000 --- a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_llama31_8b_ucx.yaml +++ /dev/null @@ -1,18 +0,0 @@ -hostname: localhost -model: llama-3.1-model/Llama-3.1-8B-Instruct -free_gpu_memory_fraction: 0.25 -backend: pytorch -disable_overlap_scheduler: true -gather_generation_logits: true -context_servers: - num_instances: 1 - tensor_parallel_size: 2 - pipeline_parallel_size: 1 - cache_transceiver_config: - backend: UCX -generation_servers: - num_instances: 1 - tensor_parallel_size: 2 - pipeline_parallel_size: 1 - cache_transceiver_config: - backend: UCX diff --git a/tests/integration/defs/disaggregated/test_disaggregated.py b/tests/integration/defs/disaggregated/test_disaggregated.py index eeb4c37f9305..f687b7ce8616 100644 --- a/tests/integration/defs/disaggregated/test_disaggregated.py +++ b/tests/integration/defs/disaggregated/test_disaggregated.py @@ -28,7 +28,6 @@ from typing import Any, Optional import aiohttp -import numpy as np import pytest import yaml from defs.common import get_free_port_in_ci as get_free_port @@ -416,8 +415,6 @@ def get_test_config(test_desc, example_dir, test_root): f"{test_configs_root}/disagg_config_cancel_stress_test.yaml", "cancel_stress_test_large": f"{test_configs_root}/disagg_config_cancel_stress_test_large.yaml", - "llama31_8b_ucx": - f"{test_configs_root}/disagg_config_ctxtp2_gentp2_llama31_8b_ucx.yaml", "mamba_conc_greater_than_mbs": f"{test_configs_root}/disagg_config_mamba_conc_greater_than_mbs.yaml", "mamba_bs1_concurrency2": @@ -4005,206 +4002,6 @@ def test_disaggregated_cancel_large_context_requests(disaggregated_test_root, cwd=llm_venv.get_working_directory()) -@pytest.mark.skip_less_device(4) -@pytest.mark.parametrize("llama_model_root", ['llama-3.1-8b-instruct'], - indirect=True) -def test_disaggregated_logprobs_serving(disaggregated_test_root, - disaggregated_example_root, llm_venv, - llama_model_root): - """Test logprobs via OpenAI API in disaggregated serving with multi-GPU TP. - - Covers the RCCA scenario (NVBug 5926823): disaggregated + streaming + logprobs, - where the context worker returns prefill result (request_type=generation_only) - to the generation worker. Ensures LogProbStorage flows correctly across the - context/gen boundary without AttributeError on cum_log_probs. - """ - - async def iter_sse_chunks(resp): - """Yield parsed JSON chunks from an OpenAI SSE stream.""" - async for line in resp.content: - decoded = line.decode("utf-8").strip() - if not decoded.startswith("data: "): - continue - data_str = decoded[len("data: "):] - if data_str == "[DONE]": - break - try: - yield json.loads(data_str) - except json.JSONDecodeError: - continue - - async def collect_streaming_logprobs(resp, api_type): - """Parse SSE stream and return (tokens, logprobs) lists.""" - tokens, logprobs = [], [] - async for chunk in iter_sse_chunks(resp): - choices = chunk.get("choices", []) - if not choices: - continue - lp_data = choices[0].get("logprobs") - if not lp_data: - continue - if api_type == "completions": - tokens.extend(lp_data.get("tokens", [])) - logprobs.extend(lp_data.get("token_logprobs", [])) - else: - for item in lp_data.get("content", []): - tokens.append(item.get("token")) - logprobs.append(item.get("logprob")) - return tokens, logprobs - - def extract_logprobs(result, api_type): - """Extract (tokens, logprobs) from non-streaming OpenAI response.""" - choices = result.get("choices", []) - assert len(choices) > 0, "Response should have choices" - if api_type == "completions": - lp_data = choices[0].get("logprobs") - assert lp_data is not None, "Response should contain logprobs" - tokens = lp_data.get("tokens", []) - logprobs = lp_data.get("token_logprobs", []) - assert len(tokens) == len(logprobs), ( - f"count mismatch: {len(logprobs)} logprobs " - f"for {len(tokens)} tokens") - return tokens, logprobs - lp_obj = choices[0].get("logprobs") - assert lp_obj is not None, "Response should contain logprobs" - content = lp_obj.get("content", []) - tokens = [item.get("token") for item in content] - logprobs = [item.get("logprob") for item in content] - return tokens, logprobs - - setup_model_symlink(llm_venv, llama_model_root, - "llama-3.1-model/Llama-3.1-8B-Instruct") - - config_file = get_test_config("llama31_8b_ucx", disaggregated_example_root, - os.path.dirname(__file__)) - - env = llm_venv._new_env.copy() - env["TRTLLM_USE_UCX_KVCACHE"] = "1" - env["UCX_TLS"] = get_ucx_tls() - ctx_workers, gen_workers, disagg_server, work_dir = [], [], None, None - config, ctx_workers, gen_workers, disagg_server, server_port, work_dir = \ - setup_disagg_cluster(config_file, env=env, - model_name=llama_model_root, - cwd=llm_venv.get_working_directory(), - server_start_timeout=600) - - server_host = config.get("hostname", "localhost") - server_url = f"http://{server_host}:{server_port}" - model_name = "llama-3.1-model/Llama-3.1-8B-Instruct" - max_tokens = 20 - timeout = aiohttp.ClientTimeout(total=120) - # Use emoji prompt to also stress-test multi-byte tokenizer handling - prompt = "I love coding 🚀 and AI." - - async def check_logprobs(): - async with aiohttp.ClientSession() as session: - for api_type in ("completions", "chat"): - url = (f"{server_url}/v1/completions" - if api_type == "completions" else - f"{server_url}/v1/chat/completions") - - def make_payload(prompt, stream, _api_type=api_type): - base = { - "max_tokens": max_tokens, - "logprobs": 1 if _api_type == "completions" else True, - "stream": stream, - "temperature": 0 - } - if _api_type == "completions": - return {"model": model_name, "prompt": prompt, **base} - return { - "model": model_name, - "messages": [{ - "role": "user", - "content": prompt - }], - **base - } - - # 1) Streaming vs non-streaming consistency check - async with session.post(url, - json=make_payload(prompt, False), - timeout=timeout) as resp: - assert resp.status == 200, \ - f"[{api_type}] non-streaming: {await resp.text()}" - ns_tokens, ns_logprobs = extract_logprobs( - await resp.json(), api_type) - - async with session.post(url, - json=make_payload(prompt, True), - timeout=timeout) as resp: - assert resp.status == 200, \ - f"[{api_type}] streaming: {await resp.text()}" - st_tokens, st_logprobs = \ - await collect_streaming_logprobs(resp, api_type) - - assert ns_tokens == st_tokens, ( - f"[{api_type}] streaming vs non-streaming tokens mismatch") - assert len(ns_logprobs) == len(st_logprobs), ( - f"[{api_type}] logprobs length: " - f"{len(ns_logprobs)} vs {len(st_logprobs)}") - # Skip position 0: the first token logprob can diverge - # between streaming and non-streaming in disaggregated mode - # due to the context/generation handoff boundary. - comparable = 0 - for i, (n, s) in enumerate( - zip(ns_logprobs, st_logprobs, strict=True)): - if i == 0 or n is None or s is None: - continue - comparable += 1 - rtol, atol = (1e-3, 1e-4) if api_type == "chat" else (1e-4, - 1e-5) - assert np.isclose(n, s, rtol=rtol, atol=atol), \ - f"[{api_type}] logprob mismatch at {i}: {n} vs {s}" - assert comparable > 0, ( - f"[{api_type}] no comparable post-handoff logprobs found") - - # 2) Chat API with top_logprobs (requires gather_generation_logits) - if api_type == "chat": - top_lp_payload = { - "model": model_name, - "messages": [{ - "role": "user", - "content": prompt - }], - "max_tokens": max_tokens, - "logprobs": True, - "top_logprobs": 3, - "stream": False, - "temperature": 0, - } - async with session.post(f"{server_url}/v1/chat/completions", - json=top_lp_payload, - timeout=timeout) as resp: - assert resp.status == 200, ( - f"[chat/top_logprobs] {resp.status}: " - f"{await resp.text()}") - result = await resp.json() - lp_obj = result["choices"][0].get("logprobs") - assert lp_obj is not None, "top_logprobs response should have logprobs" - content = lp_obj.get("content", []) - assert len( - content) > 0, "top_logprobs content should be non-empty" - for item in content: - top_lps = item.get("top_logprobs") - assert top_lps is not None and len(top_lps) > 0, ( - f"top_logprobs should be non-empty when requested: {item}" - ) - for tl in top_lps: - assert "token" in tl and "logprob" in tl, ( - f"top_logprob entry missing token/logprob: {tl}" - ) - assert tl["logprob"] <= 0.0, ( - f"top_logprob {tl['logprob']} should be <= 0") - - try: - asyncio.run(check_logprobs()) - finally: - terminate(*ctx_workers, *gen_workers, disagg_server) - if work_dir: - shutil.rmtree(work_dir, ignore_errors=True) - - @pytest.mark.skip_less_device(8) @skip_pre_blackwell @pytest.mark.parametrize("model_path", ['DeepSeek-V3-0324-FP4']) diff --git a/tests/integration/test_lists/qa/llm_function_core.txt b/tests/integration/test_lists/qa/llm_function_core.txt index b3b6a31d33cb..0904b30225da 100644 --- a/tests/integration/test_lists/qa/llm_function_core.txt +++ b/tests/integration/test_lists/qa/llm_function_core.txt @@ -864,7 +864,6 @@ disaggregated/test_disaggregated.py::test_disaggregated_genbs1[TinyLlama-1.1B-Ch disaggregated/test_disaggregated.py::test_disaggregated_gpt_oss_120b_harmony[gpt_oss/gpt-oss-120b] disaggregated/test_disaggregated.py::test_disaggregated_kv_cache_time_output[TinyLlama-1.1B-Chat-v1.0] disaggregated/test_disaggregated.py::test_disaggregated_load_balance[TinyLlama-1.1B-Chat-v1.0] -disaggregated/test_disaggregated.py::test_disaggregated_logprobs_serving[llama-3.1-8b-instruct] disaggregated/test_disaggregated.py::test_disaggregated_mamba_bs1_concurrency2 disaggregated/test_disaggregated.py::test_disaggregated_mixed[TinyLlama-1.1B-Chat-v1.0] disaggregated/test_disaggregated.py::test_disaggregated_multi_gpu[TinyLlama-1.1B-Chat-v1.0] diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index fc8bae3f0f10..38c364c159f8 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -234,7 +234,6 @@ full:H100/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mt full:H100/accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v2_kv_cache-True-True-cutlass-auto] SKIP (https://nvbugs/6529792) full:H100/accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_softmax_attention_4gpus[target_sparsity_0.9-fp8kv=False] SKIP (https://nvbugs/6523809) full:H100/accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16[tp1-CUTLASS] SKIP (https://nvbugs/6273850) -full:H100/disaggregated/test_disaggregated.py::test_disaggregated_logprobs_serving[llama-3.1-8b-instruct] SKIP (https://nvbugs/6275959) full:H100/disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-qwen3_32b_fp8_stress] SKIP (https://nvbugs/6312828) full:H100_PCIe/unittest/llmapi/test_llm_pytorch.py::test_llama_7b_multi_lora_evict_and_reload_lora_gpu_cache SKIP (https://nvbugs/5682551) full:H20/accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=False] SKIP (https://nvbugs/6345827) From 9143f9d0cb78020e6f0f8ed13cd1155b6547ad08 Mon Sep 17 00:00:00 2001 From: Shixiaowei02 <39303645+Shixiaowei02@users.noreply.github.com> Date: Sun, 2 Aug 2026 20:14:16 -0700 Subject: [PATCH 5/7] [None][test] Keep disagg logprobs regression test, on a DEFAULT-backend config Review follow-up on #16614 (yingguo-trt, fredricz-20070104): the consolidation dropped test_disaggregated_logprobs_serving, which is the resident regression guard for NVBug 5926823 (disagg + streaming + logprobs). Its API-level coverage is unique -- streaming vs. non-streaming logprobs consistency across both the completions and chat APIs, plus chat top_logprobs across the ctx/gen boundary. test_disaggregated_single_gpu.py::test_disaggregated_logprobs only covers the core prefill->decode propagation, so deleting this would be a test escape. The test was never UCX-specific; it only reused the UCX yaml. Restore it on a new DEFAULT-backend config and drop the explicit TRTLLM_USE_UCX_KVCACHE pin, which keeps the de-UCX-specialization goal of this PR intact. UCX_TLS is kept, matching every other non-pinned test in this file. The waive for the open bug (NVBugs 6275959, fix in flight in #15221) is restored with it. Restores the numpy import: np.isclose in this test is its only user. Signed-off-by: Shixiaowei02 <39303645+Shixiaowei02@users.noreply.github.com> Co-authored-by: Claude --- ...isagg_config_ctxtp2_gentp2_llama31_8b.yaml | 18 ++ .../defs/disaggregated/test_disaggregated.py | 202 ++++++++++++++++++ tests/integration/test_lists/waives.txt | 1 + 3 files changed, 221 insertions(+) create mode 100644 tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_llama31_8b.yaml diff --git a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_llama31_8b.yaml b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_llama31_8b.yaml new file mode 100644 index 000000000000..487abd0d600e --- /dev/null +++ b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_llama31_8b.yaml @@ -0,0 +1,18 @@ +hostname: localhost +model: llama-3.1-model/Llama-3.1-8B-Instruct +free_gpu_memory_fraction: 0.25 +backend: pytorch +disable_overlap_scheduler: true +gather_generation_logits: true +context_servers: + num_instances: 1 + tensor_parallel_size: 2 + pipeline_parallel_size: 1 + cache_transceiver_config: + backend: DEFAULT +generation_servers: + num_instances: 1 + tensor_parallel_size: 2 + pipeline_parallel_size: 1 + cache_transceiver_config: + backend: DEFAULT diff --git a/tests/integration/defs/disaggregated/test_disaggregated.py b/tests/integration/defs/disaggregated/test_disaggregated.py index f687b7ce8616..12a3865d7ff4 100644 --- a/tests/integration/defs/disaggregated/test_disaggregated.py +++ b/tests/integration/defs/disaggregated/test_disaggregated.py @@ -28,6 +28,7 @@ from typing import Any, Optional import aiohttp +import numpy as np import pytest import yaml from defs.common import get_free_port_in_ci as get_free_port @@ -415,6 +416,8 @@ def get_test_config(test_desc, example_dir, test_root): f"{test_configs_root}/disagg_config_cancel_stress_test.yaml", "cancel_stress_test_large": f"{test_configs_root}/disagg_config_cancel_stress_test_large.yaml", + "llama31_8b": + f"{test_configs_root}/disagg_config_ctxtp2_gentp2_llama31_8b.yaml", "mamba_conc_greater_than_mbs": f"{test_configs_root}/disagg_config_mamba_conc_greater_than_mbs.yaml", "mamba_bs1_concurrency2": @@ -4002,6 +4005,205 @@ def test_disaggregated_cancel_large_context_requests(disaggregated_test_root, cwd=llm_venv.get_working_directory()) +@pytest.mark.skip_less_device(4) +@pytest.mark.parametrize("llama_model_root", ['llama-3.1-8b-instruct'], + indirect=True) +def test_disaggregated_logprobs_serving(disaggregated_test_root, + disaggregated_example_root, llm_venv, + llama_model_root): + """Test logprobs via OpenAI API in disaggregated serving with multi-GPU TP. + + Covers the RCCA scenario (NVBug 5926823): disaggregated + streaming + logprobs, + where the context worker returns prefill result (request_type=generation_only) + to the generation worker. Ensures LogProbStorage flows correctly across the + context/gen boundary without AttributeError on cum_log_probs. + """ + + async def iter_sse_chunks(resp): + """Yield parsed JSON chunks from an OpenAI SSE stream.""" + async for line in resp.content: + decoded = line.decode("utf-8").strip() + if not decoded.startswith("data: "): + continue + data_str = decoded[len("data: "):] + if data_str == "[DONE]": + break + try: + yield json.loads(data_str) + except json.JSONDecodeError: + continue + + async def collect_streaming_logprobs(resp, api_type): + """Parse SSE stream and return (tokens, logprobs) lists.""" + tokens, logprobs = [], [] + async for chunk in iter_sse_chunks(resp): + choices = chunk.get("choices", []) + if not choices: + continue + lp_data = choices[0].get("logprobs") + if not lp_data: + continue + if api_type == "completions": + tokens.extend(lp_data.get("tokens", [])) + logprobs.extend(lp_data.get("token_logprobs", [])) + else: + for item in lp_data.get("content", []): + tokens.append(item.get("token")) + logprobs.append(item.get("logprob")) + return tokens, logprobs + + def extract_logprobs(result, api_type): + """Extract (tokens, logprobs) from non-streaming OpenAI response.""" + choices = result.get("choices", []) + assert len(choices) > 0, "Response should have choices" + if api_type == "completions": + lp_data = choices[0].get("logprobs") + assert lp_data is not None, "Response should contain logprobs" + tokens = lp_data.get("tokens", []) + logprobs = lp_data.get("token_logprobs", []) + assert len(tokens) == len(logprobs), ( + f"count mismatch: {len(logprobs)} logprobs " + f"for {len(tokens)} tokens") + return tokens, logprobs + lp_obj = choices[0].get("logprobs") + assert lp_obj is not None, "Response should contain logprobs" + content = lp_obj.get("content", []) + tokens = [item.get("token") for item in content] + logprobs = [item.get("logprob") for item in content] + return tokens, logprobs + + setup_model_symlink(llm_venv, llama_model_root, + "llama-3.1-model/Llama-3.1-8B-Instruct") + + config_file = get_test_config("llama31_8b", disaggregated_example_root, + os.path.dirname(__file__)) + + env = llm_venv._new_env.copy() + env["UCX_TLS"] = get_ucx_tls() + ctx_workers, gen_workers, disagg_server, work_dir = [], [], None, None + config, ctx_workers, gen_workers, disagg_server, server_port, work_dir = \ + setup_disagg_cluster(config_file, env=env, + model_name=llama_model_root, + cwd=llm_venv.get_working_directory(), + server_start_timeout=600) + + server_host = config.get("hostname", "localhost") + server_url = f"http://{server_host}:{server_port}" + model_name = "llama-3.1-model/Llama-3.1-8B-Instruct" + max_tokens = 20 + timeout = aiohttp.ClientTimeout(total=120) + # Use emoji prompt to also stress-test multi-byte tokenizer handling + prompt = "I love coding 🚀 and AI." + + async def check_logprobs(): + async with aiohttp.ClientSession() as session: + for api_type in ("completions", "chat"): + url = (f"{server_url}/v1/completions" + if api_type == "completions" else + f"{server_url}/v1/chat/completions") + + def make_payload(prompt, stream, _api_type=api_type): + base = { + "max_tokens": max_tokens, + "logprobs": 1 if _api_type == "completions" else True, + "stream": stream, + "temperature": 0 + } + if _api_type == "completions": + return {"model": model_name, "prompt": prompt, **base} + return { + "model": model_name, + "messages": [{ + "role": "user", + "content": prompt + }], + **base + } + + # 1) Streaming vs non-streaming consistency check + async with session.post(url, + json=make_payload(prompt, False), + timeout=timeout) as resp: + assert resp.status == 200, \ + f"[{api_type}] non-streaming: {await resp.text()}" + ns_tokens, ns_logprobs = extract_logprobs( + await resp.json(), api_type) + + async with session.post(url, + json=make_payload(prompt, True), + timeout=timeout) as resp: + assert resp.status == 200, \ + f"[{api_type}] streaming: {await resp.text()}" + st_tokens, st_logprobs = \ + await collect_streaming_logprobs(resp, api_type) + + assert ns_tokens == st_tokens, ( + f"[{api_type}] streaming vs non-streaming tokens mismatch") + assert len(ns_logprobs) == len(st_logprobs), ( + f"[{api_type}] logprobs length: " + f"{len(ns_logprobs)} vs {len(st_logprobs)}") + # Skip position 0: the first token logprob can diverge + # between streaming and non-streaming in disaggregated mode + # due to the context/generation handoff boundary. + comparable = 0 + for i, (n, s) in enumerate( + zip(ns_logprobs, st_logprobs, strict=True)): + if i == 0 or n is None or s is None: + continue + comparable += 1 + rtol, atol = (1e-3, 1e-4) if api_type == "chat" else (1e-4, + 1e-5) + assert np.isclose(n, s, rtol=rtol, atol=atol), \ + f"[{api_type}] logprob mismatch at {i}: {n} vs {s}" + assert comparable > 0, ( + f"[{api_type}] no comparable post-handoff logprobs found") + + # 2) Chat API with top_logprobs (requires gather_generation_logits) + if api_type == "chat": + top_lp_payload = { + "model": model_name, + "messages": [{ + "role": "user", + "content": prompt + }], + "max_tokens": max_tokens, + "logprobs": True, + "top_logprobs": 3, + "stream": False, + "temperature": 0, + } + async with session.post(f"{server_url}/v1/chat/completions", + json=top_lp_payload, + timeout=timeout) as resp: + assert resp.status == 200, ( + f"[chat/top_logprobs] {resp.status}: " + f"{await resp.text()}") + result = await resp.json() + lp_obj = result["choices"][0].get("logprobs") + assert lp_obj is not None, "top_logprobs response should have logprobs" + content = lp_obj.get("content", []) + assert len( + content) > 0, "top_logprobs content should be non-empty" + for item in content: + top_lps = item.get("top_logprobs") + assert top_lps is not None and len(top_lps) > 0, ( + f"top_logprobs should be non-empty when requested: {item}" + ) + for tl in top_lps: + assert "token" in tl and "logprob" in tl, ( + f"top_logprob entry missing token/logprob: {tl}" + ) + assert tl["logprob"] <= 0.0, ( + f"top_logprob {tl['logprob']} should be <= 0") + + try: + asyncio.run(check_logprobs()) + finally: + terminate(*ctx_workers, *gen_workers, disagg_server) + if work_dir: + shutil.rmtree(work_dir, ignore_errors=True) + + @pytest.mark.skip_less_device(8) @skip_pre_blackwell @pytest.mark.parametrize("model_path", ['DeepSeek-V3-0324-FP4']) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 38c364c159f8..fc8bae3f0f10 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -234,6 +234,7 @@ full:H100/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mt full:H100/accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_1gpu[v2_kv_cache-True-True-cutlass-auto] SKIP (https://nvbugs/6529792) full:H100/accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B_Instruct_2507::test_skip_softmax_attention_4gpus[target_sparsity_0.9-fp8kv=False] SKIP (https://nvbugs/6523809) full:H100/accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16[tp1-CUTLASS] SKIP (https://nvbugs/6273850) +full:H100/disaggregated/test_disaggregated.py::test_disaggregated_logprobs_serving[llama-3.1-8b-instruct] SKIP (https://nvbugs/6275959) full:H100/disaggregated/test_disaggregated.py::test_disaggregated_stress_test[input8k-output1k-conc512-qwen3_32b_fp8_stress] SKIP (https://nvbugs/6312828) full:H100_PCIe/unittest/llmapi/test_llm_pytorch.py::test_llama_7b_multi_lora_evict_and_reload_lora_gpu_cache SKIP (https://nvbugs/5682551) full:H20/accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_dtype[mtp_nextn=2-overlap_scheduler=False] SKIP (https://nvbugs/6345827) From 723e88918bb288155c8d4cf91f66012e921b0e01 Mon Sep 17 00:00:00 2001 From: Xiaowei Shi <39303645+Shixiaowei02@users.noreply.github.com> Date: Thu, 6 Aug 2026 03:05:55 -0700 Subject: [PATCH 6/7] [None][test] Keep the disagg logprobs test scheduled, fix stale waives test_disaggregated_logprobs_serving is the regression guard for NVBug 5926823 (disagg + streaming + logprobs). The migration to the DEFAULT-backend config kept the test function and its waive, but dropped its entry from qa/llm_function_core.txt, leaving it in no test list at all -- so lifting the 6275959 waive later would not have brought it back. Restore the entry; the waive stays. validate-test-lists was already reporting this as a waive with no active test list. Two more entries that hook rejected, both from this branch's parametrize consolidation rather than from review: - test_guided_decoding[llguidance-mtp_nextn=2]: the backend list is now ["xgrammar"] only, so this ID no longer exists. - test_auto_dtype[False-True-True-True]: the four boolean parametrize decorators became one (ctx,gen) pair with ids block_reuse/no_block_reuse, and the waived combination was the mixed ctx=True/gen=False one, which no longer exists. Both waives point at test IDs that cannot be collected, so they are removed. If 6525893 still reproduces on a surviving test_auto_dtype variant, its waive needs re-pointing rather than deleting. Also record why DEFAULT is used instead of NIXL: launch_disaggregated_llm still sets TRTLLM_USE_UCX_KVCACHE=1 for every backend but NIXL, so dropping the per-test UCX pinning leaves transport coverage unchanged. Signed-off-by: Xiaowei Shi <39303645+Shixiaowei02@users.noreply.github.com> --- tests/integration/defs/accuracy/test_disaggregated_serving.py | 3 +++ tests/integration/test_lists/qa/llm_function_core.txt | 1 + tests/integration/test_lists/waives.txt | 2 -- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/integration/defs/accuracy/test_disaggregated_serving.py b/tests/integration/defs/accuracy/test_disaggregated_serving.py index fb3d98d8b072..b128ff88eeb4 100644 --- a/tests/integration/defs/accuracy/test_disaggregated_serving.py +++ b/tests/integration/defs/accuracy/test_disaggregated_serving.py @@ -1142,6 +1142,9 @@ def test_auto_dtype_with_helix(self, comms_medium, cuda_graph_config, "kv_cache_config": kv_cache_config, "enable_chunked_prefill": False, "cuda_graph_config": None, + # DEFAULT drops the per-test UCX pinning but still runs UCX, since + # launch_disaggregated_llm sets TRTLLM_USE_UCX_KVCACHE=1 for every + # backend but NIXL. Transport coverage is unchanged by this move. "cache_transceiver_config": { "backend": "DEFAULT", "max_tokens_in_buffer": 8192, diff --git a/tests/integration/test_lists/qa/llm_function_core.txt b/tests/integration/test_lists/qa/llm_function_core.txt index 0904b30225da..b3b6a31d33cb 100644 --- a/tests/integration/test_lists/qa/llm_function_core.txt +++ b/tests/integration/test_lists/qa/llm_function_core.txt @@ -864,6 +864,7 @@ disaggregated/test_disaggregated.py::test_disaggregated_genbs1[TinyLlama-1.1B-Ch disaggregated/test_disaggregated.py::test_disaggregated_gpt_oss_120b_harmony[gpt_oss/gpt-oss-120b] disaggregated/test_disaggregated.py::test_disaggregated_kv_cache_time_output[TinyLlama-1.1B-Chat-v1.0] disaggregated/test_disaggregated.py::test_disaggregated_load_balance[TinyLlama-1.1B-Chat-v1.0] +disaggregated/test_disaggregated.py::test_disaggregated_logprobs_serving[llama-3.1-8b-instruct] disaggregated/test_disaggregated.py::test_disaggregated_mamba_bs1_concurrency2 disaggregated/test_disaggregated.py::test_disaggregated_mixed[TinyLlama-1.1B-Chat-v1.0] disaggregated/test_disaggregated.py::test_disaggregated_multi_gpu[TinyLlama-1.1B-Chat-v1.0] diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index fc8bae3f0f10..85d8d20a15c7 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -189,7 +189,6 @@ full:DGX_B200/perf/test_perf_sanity.py::test_e2e[aggr_upload-gemma4_26b_a4b_nvfp full:DGX_B200/perf/test_perf_sanity.py::test_e2e[aggr_upload-host_perf_llama8b_spec_decode-llama8b_spec_bs1_128_128] SKIP (https://nvbugs/6571408) full:DGX_H200/accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_fp8_blockscale[disable_skip_indexer] SKIP (https://nvbugs/6476233) full:DGX_H200/accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_fp8_blockscale[latency_default] SKIP (https://nvbugs/6476233) -full:GB200/accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-True-True-True] SKIP (https://nvbugs/6525893) full:GB200/accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy SKIP (https://nvbugs/6276923) full:GB200/accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy_contention_opt SKIP (https://nvbugs/6276923) full:GB200/accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy_mode_b_overlap SKIP (https://nvbugs/6276923) @@ -258,7 +257,6 @@ full:RTX_6000D/accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_bf16_ full:RTX_6000D/accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_fp8[enable_block_reuse=False] SKIP (https://nvbugs/6313076) full:RTX_6000D/accuracy/test_llm_api_pytorch.py::TestQwen3_5_4B::test_dflash SKIP (https://nvbugs/6273850) full:RTX_6000D/accuracy/test_llm_api_pytorch.py::TestQwen3_5_4B::test_fp8 SKIP (https://nvbugs/6273850) -full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_guided_decoding[llguidance-mtp_nextn=2] SKIP (https://nvbugs/6473374) full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True] SKIP (https://nvbugs/6313072) full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=False] SKIP (https://nvbugs/6313072) full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=True] SKIP (https://nvbugs/6313072) From 125fa224c9f95d25ed69dc3d75185168260f9def Mon Sep 17 00:00:00 2001 From: Xiaowei Shi <39303645+Shixiaowei02@users.noreply.github.com> Date: Sun, 9 Aug 2026 19:48:29 -0700 Subject: [PATCH 7/7] [None][test] Keep the asymmetric block-reuse case, drop rebase artifacts - test_auto_dtype: zipping the two block-reuse flags to [(True, True), (False, False)] kept only the symmetric pairs. ctx_enable_block_reuse=True with gen_enable_block_reuse=False is the one combination where the servers disagree about which blocks are resident, so it takes a different transfer path; add it back as `ctx_block_reuse_only`. Three runs instead of two, still a 5x cut from sixteen. The GB200 waive for nvbugs/6525893 -- which tracked exactly that combination under the old [False-True-True-True] id -- is re-pointed at the new id rather than dropped. - waives.txt: drop the two H100 TestQwen3_5_*_VL::test_auto_dtype entries. They are unrelated to this consolidation and came in as a rebase artifact. - test_disaggregated_logprobs_serving: with TRTLLM_USE_UCX_KVCACHE=1 gone and the config on backend DEFAULT, this harness no longer selects UCX, so the leftover UCX_TLS export does nothing. (The accuracy harness is different: launch_disaggregated_llm still sets TRTLLM_USE_UCX_KVCACHE=1 for every backend but NIXL, which is why its DEFAULT configs keep UCX coverage.) Signed-off-by: Xiaowei Shi <39303645+Shixiaowei02@users.noreply.github.com> --- .../defs/accuracy/test_disaggregated_serving.py | 11 ++++++++--- .../defs/disaggregated/test_disaggregated.py | 3 --- tests/integration/test_lists/qa/llm_function_core.txt | 1 + tests/integration/test_lists/test-db/l0_dgx_h100.yml | 1 + tests/integration/test_lists/waives.txt | 1 + 5 files changed, 11 insertions(+), 6 deletions(-) diff --git a/tests/integration/defs/accuracy/test_disaggregated_serving.py b/tests/integration/defs/accuracy/test_disaggregated_serving.py index b128ff88eeb4..6511f0c43425 100644 --- a/tests/integration/defs/accuracy/test_disaggregated_serving.py +++ b/tests/integration/defs/accuracy/test_disaggregated_serving.py @@ -624,9 +624,14 @@ class TestLlama3_1_8BInstruct(LlmapiAccuracyTestHarness): @skip_pre_hopper @pytest.mark.skip_less_device(2) # overlap scheduler is token-invariant (unit-tested); only block-reuse changes which KV is transferred - @pytest.mark.parametrize("ctx_enable_block_reuse,gen_enable_block_reuse", - [(True, True), (False, False)], - ids=["block_reuse", "no_block_reuse"]) + # The mismatched pair is kept on purpose: it is the only combination where + # the two servers disagree about which blocks are already resident, so it + # exercises a different transfer path than either symmetric case. + @pytest.mark.parametrize( + "ctx_enable_block_reuse,gen_enable_block_reuse", [(True, True), + (True, False), + (False, False)], + ids=["block_reuse", "ctx_block_reuse_only", "no_block_reuse"]) def test_auto_dtype(self, ctx_enable_block_reuse, gen_enable_block_reuse): ctx_server_config = { "disable_overlap_scheduler": False, diff --git a/tests/integration/defs/disaggregated/test_disaggregated.py b/tests/integration/defs/disaggregated/test_disaggregated.py index 12a3865d7ff4..6789ee0c91a7 100644 --- a/tests/integration/defs/disaggregated/test_disaggregated.py +++ b/tests/integration/defs/disaggregated/test_disaggregated.py @@ -2038,8 +2038,6 @@ def test_disaggregated_deepseek_v3_lite_fp8_nixl(disaggregated_test_root, cwd=llm_venv.get_working_directory()) - - @skip_no_hopper @pytest.mark.skip_less_device(4) @pytest.mark.parametrize("deepseek_v3_model_root", ['DeepSeek-V3-Lite-fp8'], @@ -4079,7 +4077,6 @@ def extract_logprobs(result, api_type): os.path.dirname(__file__)) env = llm_venv._new_env.copy() - env["UCX_TLS"] = get_ucx_tls() ctx_workers, gen_workers, disagg_server, work_dir = [], [], None, None config, ctx_workers, gen_workers, disagg_server, server_port, work_dir = \ setup_disagg_cluster(config_file, env=env, diff --git a/tests/integration/test_lists/qa/llm_function_core.txt b/tests/integration/test_lists/qa/llm_function_core.txt index b3b6a31d33cb..7fdecf023c9a 100644 --- a/tests/integration/test_lists/qa/llm_function_core.txt +++ b/tests/integration/test_lists/qa/llm_function_core.txt @@ -20,6 +20,7 @@ accuracy/test_disaggregated_serving.py::TestGPTOSS::test_kv_cache_v2_nixl_python accuracy/test_disaggregated_serving.py::TestGLM52NVFP4::test_nvfp4_nixl[cache_mgr_v1] accuracy/test_disaggregated_serving.py::TestKimiK25::test_nvfp4 accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[block_reuse] +accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[ctx_block_reuse_only] accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[no_block_reuse] accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=1-ctx_pp=2] accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_ctx_pp_gen_tp_asymmetric[GSM8K-gen_tp=1-ctx_pp=4] diff --git a/tests/integration/test_lists/test-db/l0_dgx_h100.yml b/tests/integration/test_lists/test-db/l0_dgx_h100.yml index ffefc4a3574f..46d80430ffe9 100644 --- a/tests/integration/test_lists/test-db/l0_dgx_h100.yml +++ b/tests/integration/test_lists/test-db/l0_dgx_h100.yml @@ -42,6 +42,7 @@ l0_dgx_h100: - accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_kv_cache_v2_nixl_python[cache_mgr_v1] - accuracy/test_disaggregated_serving.py::TestGemma3_1BInstruct::test_kv_cache_v2_nixl_python[cache_mgr_v2] - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[block_reuse] + - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[ctx_block_reuse_only] - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[no_block_reuse] - unittest/llmapi/apps/test_disagg_serving_perf_metrics.py - disaggregated/test_disaggregated.py::test_disaggregated_cancel_large_context_requests[DeepSeek-V3-Lite-bf16] diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 85d8d20a15c7..15525a5359df 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -189,6 +189,7 @@ full:DGX_B200/perf/test_perf_sanity.py::test_e2e[aggr_upload-gemma4_26b_a4b_nvfp full:DGX_B200/perf/test_perf_sanity.py::test_e2e[aggr_upload-host_perf_llama8b_spec_decode-llama8b_spec_bs1_128_128] SKIP (https://nvbugs/6571408) full:DGX_H200/accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_fp8_blockscale[disable_skip_indexer] SKIP (https://nvbugs/6476233) full:DGX_H200/accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_fp8_blockscale[latency_default] SKIP (https://nvbugs/6476233) +full:GB200/accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[ctx_block_reuse_only] SKIP (https://nvbugs/6525893) full:GB200/accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy SKIP (https://nvbugs/6276923) full:GB200/accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy_contention_opt SKIP (https://nvbugs/6276923) full:GB200/accuracy/test_dwdp_disaggregated_serving.py::TestDwdpDeepSeekV3Lite::test_dwdp_accuracy_mode_b_overlap SKIP (https://nvbugs/6276923)