From c00006e1760f3fcee6d1b325ec5461a684e2e4df Mon Sep 17 00:00:00 2001 From: wooway777 Date: Thu, 16 Jul 2026 08:00:18 +0000 Subject: [PATCH] fix: resupport ep in inference server --- python/infinilm/server/inference_server.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/infinilm/server/inference_server.py b/python/infinilm/server/inference_server.py index 242593e2..929c9952 100644 --- a/python/infinilm/server/inference_server.py +++ b/python/infinilm/server/inference_server.py @@ -100,6 +100,7 @@ def __init__( tensor_parallel_size: int = 1, moe_ep_backend: str = "disabled", moe_ep_size: int = 1, + skip_legacy_moe: bool = False, cache_type: str = "paged", max_tokens: int = 4096, max_batch_size: int = 16, @@ -127,6 +128,7 @@ def __init__( tensor_parallel_size: Number of devices for tensor parallelism. moe_ep_backend: MoE expert-parallel backend. moe_ep_size: MoE expert-parallel size. + skip_legacy_moe: Whether to use the fused Qwen3 MoE implementation. cache_type: Cache type ('paged' or 'static'). max_tokens: Default maximum tokens to generate. max_batch_size: Maximum batch size for inference (only for paged cache). @@ -153,6 +155,7 @@ def __init__( self.tensor_parallel_size = tensor_parallel_size self.moe_ep_backend = moe_ep_backend self.moe_ep_size = moe_ep_size + self.skip_legacy_moe = skip_legacy_moe self.cache_type = cache_type self.max_tokens = max_tokens self.max_batch_size = max_batch_size @@ -192,6 +195,7 @@ async def lifespan(app: FastAPI): tensor_parallel_size=self.tensor_parallel_size, moe_ep_backend=self.moe_ep_backend, moe_ep_size=self.moe_ep_size, + skip_legacy_moe=self.skip_legacy_moe, cache_type=self.cache_type, max_batch_size=self.max_batch_size, max_tokens=self.max_tokens, @@ -607,6 +611,7 @@ def main(): tensor_parallel_size=cfg.tp, moe_ep_backend=moe_ep_backend, moe_ep_size=ep, + skip_legacy_moe=cfg.skip_legacy_moe, cache_type="paged" if cfg.enable_paged_attn else "static", max_tokens=cfg.max_new_tokens, max_batch_size=cfg.max_batch_size,