Skip to content

fix: resupport ep in inference server #505

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
wooway777 merged 1 commit into from
Jul 16, 2026
Merged

fix: resupport ep in inference server #505

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions python/infinilm/server/inference_server.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ def __init__(
tensor_parallel_size: int = 1,
moe_ep_backend: str = "disabled",
moe_ep_size: int = 1,
skip_legacy_moe: bool = False,
cache_type: str = "paged",
max_tokens: int = 4096,
max_batch_size: int = 16,
Expand Down Expand Up @@ -127,6 +128,7 @@ def __init__(
tensor_parallel_size: Number of devices for tensor parallelism.
moe_ep_backend: MoE expert-parallel backend.
moe_ep_size: MoE expert-parallel size.
skip_legacy_moe: Whether to use the fused Qwen3 MoE implementation.
cache_type: Cache type ('paged' or 'static').
max_tokens: Default maximum tokens to generate.
max_batch_size: Maximum batch size for inference (only for paged cache).
Expand All @@ -153,6 +155,7 @@ def __init__(
self.tensor_parallel_size = tensor_parallel_size
self.moe_ep_backend = moe_ep_backend
self.moe_ep_size = moe_ep_size
self.skip_legacy_moe = skip_legacy_moe
self.cache_type = cache_type
self.max_tokens = max_tokens
self.max_batch_size = max_batch_size
Expand Down Expand Up @@ -192,6 +195,7 @@ async def lifespan(app: FastAPI):
tensor_parallel_size=self.tensor_parallel_size,
moe_ep_backend=self.moe_ep_backend,
moe_ep_size=self.moe_ep_size,
skip_legacy_moe=self.skip_legacy_moe,
cache_type=self.cache_type,
max_batch_size=self.max_batch_size,
max_tokens=self.max_tokens,
Expand Down Expand Up @@ -607,6 +611,7 @@ def main():
tensor_parallel_size=cfg.tp,
moe_ep_backend=moe_ep_backend,
moe_ep_size=ep,
skip_legacy_moe=cfg.skip_legacy_moe,
cache_type="paged" if cfg.enable_paged_attn else "static",
max_tokens=cfg.max_new_tokens,
max_batch_size=cfg.max_batch_size,
Expand Down
Loading