From ca493f11fe5b23b259ab8c4547edcece0cc067b0 Mon Sep 17 00:00:00 2001
From: exzile <joeypongallo@gmail.com>
Date: Fri, 26 Jun 2026 15:54:45 -0400
Subject: [PATCH 1/5] Support add_generation_prompt request parameter for chat
 completions

The chat template was always rendered with add_generation_prompt=true,
hardcoded in every servable. This exposes an optional add_generation_prompt
field (bool, default true) on the /v3/chat/completions request, matching
HF transformers and vLLM. When false, the trailing generation prompt is
omitted, which is the building block for assistant prefill.

- Parse add_generation_prompt in the request (openai_api_handler.cpp) and
  store it on the request struct (openai_request.hpp).
- Honor it in all chat-template application sites: the MINJA path (LLM and
  VLM continuous batching, legacy) and the Python-Jinja path (read from the
  request body and pass into the template render).
- Add tests covering default (generation prompt added) and false
  (generation prompt omitted, assistant message preserved).

Verified end-to-end on the MINJA path with HuggingFaceTB/SmolLM2-360M-Instruct:
default renders a trailing "<|im_start|>assistant", add_generation_prompt=false
omits it.

Note: true assistant prefill (continue_final_message - continuing from the
final assistant message without closing it) is a separate control and is left
as a follow-up.

Implements #3877

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 docs/model_server_rest_api_chat.md            |  1 +
 src/llm/apis/openai_api_handler.cpp           | 12 +++++++
 src/llm/apis/openai_request.hpp               |  3 ++
 .../chat_template_processor.cpp               |  4 +--
 src/llm/io_processing/input_request.hpp       |  1 +
 src/llm/py_jinja_template_processor.cpp       |  8 +++--
 src/test/llm/llmtemplate_test.cpp             | 32 +++++++++++++++++++
 7 files changed, 57 insertions(+), 4 deletions(-)

diff --git a/docs/model_server_rest_api_chat.md b/docs/model_server_rest_api_chat.md
index 951694f8f4..332fb330a2 100644
--- a/docs/model_server_rest_api_chat.md
+++ b/docs/model_server_rest_api_chat.md
@@ -222,6 +222,7 @@ Some parameters, especially related to sampling (like `temperature`, `top_p` etc
 | response_format | ✅ | ✅ | ✅ | object | An object specifying the format that the model must output. Setting to `{ "type": "json_schema", "json_schema": {...} }` enables Structured Outputs which ensures the model will match your supplied JSON schema according to [OpenAI reference](https://platform.openai.com/docs/api-reference/chat/create#chat-create-response_format). Learn more in the [Structured Outputs demo](../demos/continuous_batching/structured_output/README.md). Additionally, `response_format` can accept [XGrammar structural tags format](https://github.com/mlc-ai/xgrammar/blob/v0.1.26/docs/tutorials/structural_tag.md#format-types) (not part of OpenAI API). For example: `{ "type": "const_string", "value": "Hello World!" }`. **Note** that if model server fails to process the format, the request will still be processed, but the format will not be imposed. |
 | chat_template_kwargs | ✅ | ❌ | ✅ |  object | Enables passing additional parameters to chat template engine. Example `{"enable_thinking": false}`. Note that values like `messages`, `eos_token`, `bos_token` etc. are provided natively to the template engine, so including them in `chat_template_kwargs` will cause error. |
 | skip_special_tokens | ✅ | ❌ | ✅ | bool (default: `true`) | Whether to remove special tokens (e.g. `<\|endoftext\|>`, `<\|im_end\|>`) from the generated output. Set to `false` to include them, which is useful when the model uses special tokens to encode structured information (e.g. bounding boxes, reasoning markers). When `false`, any tool or reasoning parser configured on the endpoint is silently disabled for the request, so the raw token stream is returned. This option works with most detokenizers exported with OpenVINO Tokenizers 2024.5 or later, unless they are based on custom ops. |
+| add_generation_prompt | ✅ | ❌ | ✅ | bool (default: `true`) | Whether to append the chat template's generation prompt (the marker that signals the model to start a new assistant turn). Set to `false` to render the conversation without a trailing generation prompt — useful for assistant prefill where the final `assistant` message should be continued rather than treated as a completed turn. Applies to both the Python-Jinja and MINJA chat template paths. |
 
 #### Beam search sampling specific
 | Param | OpenVINO Model Server | OpenAI /chat/completions API | vLLM Serving Sampling Params | Type | Description |
diff --git a/src/llm/apis/openai_api_handler.cpp b/src/llm/apis/openai_api_handler.cpp
index b8b9776199..15dd44895c 100644
--- a/src/llm/apis/openai_api_handler.cpp
+++ b/src/llm/apis/openai_api_handler.cpp
@@ -498,6 +498,7 @@ absl::StatusOr<InputRequest> OpenAIApiHandler::extractInputRequest(GenerationCon
     }
     InputRequest req;
     req.generationConfig = configBuilder.getConfig();
+    req.addGenerationPrompt = request.addGenerationPrompt.value_or(true);
     if (endpoint == Endpoint::COMPLETIONS) {
         req.input = request.prompt.value_or("");
     } else {
@@ -685,6 +686,17 @@ absl::Status OpenAIApiHandler::parseCommonPart(std::optional<uint32_t> maxTokens
         request.ignoreEOS = it->value.GetBool();
     }
 
+    // add_generation_prompt: bool; optional - defaults to true
+    // Extension, unsupported by OpenAI API, however supported by HF transformers and vLLM.
+    // When false, the chat template is rendered without a trailing generation prompt
+    // so a final assistant message can be continued as a prefix (assistant prefill).
+    it = doc.FindMember("add_generation_prompt");
+    if (it != doc.MemberEnd() && !it->value.IsNull()) {
+        if (!it->value.IsBool())
+            return absl::InvalidArgumentError("add_generation_prompt accepts values true or false");
+        request.addGenerationPrompt = it->value.GetBool();
+    }
+
     // max_tokens: uint; optional
     // Common part checked here, specific parts are checked in parseCompletionsPart and parseChatCompletionsPart
     // TODO: Deprecated - this will need to be removed in the future
diff --git a/src/llm/apis/openai_request.hpp b/src/llm/apis/openai_request.hpp
index 5b433a7a0b..faf100f94a 100644
--- a/src/llm/apis/openai_request.hpp
+++ b/src/llm/apis/openai_request.hpp
@@ -48,6 +48,9 @@ struct OpenAIRequest {
     int logprobschat{0};
     bool echo{false};
     std::optional<bool> ignoreEOS{std::nullopt};
+    // When false, the chat template is rendered without a trailing generation prompt
+    // (e.g. for assistant prefill). Defaults to true. Extension supported by HF/vLLM.
+    std::optional<bool> addGenerationPrompt{std::nullopt};
     std::optional<std::set<std::string>> stop{std::nullopt};
     std::optional<bool> includeStopStrInOutput{std::nullopt};
     std::optional<int> numReturnSequences{std::nullopt};  // effective for beam search and multinomial decoding
diff --git a/src/llm/io_processing/input_processors/chat_template_processor.cpp b/src/llm/io_processing/input_processors/chat_template_processor.cpp
index 8966cb33f8..619570a795 100644
--- a/src/llm/io_processing/input_processors/chat_template_processor.cpp
+++ b/src/llm/io_processing/input_processors/chat_template_processor.cpp
@@ -67,7 +67,7 @@ absl::Status ChatTemplateProcessor::process(InputRequest& req) {
         SPDLOG_LOGGER_TRACE(llm_calculator_logger, "chatHistory.get_extra_context(): {}", chatHistory.get_extra_context().to_json_string());
         SPDLOG_LOGGER_TRACE(llm_calculator_logger, "tools: {}", chatHistory.get_tools().empty() ? std::string("<none>") : chatHistory.get_tools().to_json_string());
         SPDLOG_LOGGER_TRACE(llm_calculator_logger, "chatTemplateKwargs: {}", chatHistory.get_extra_context().empty() ? std::string("<none>") : chatHistory.get_extra_context().to_json_string());
-        SPDLOG_LOGGER_TRACE(llm_calculator_logger, "addGenerationPrompt: {}", true);
+        SPDLOG_LOGGER_TRACE(llm_calculator_logger, "addGenerationPrompt: {}", req.addGenerationPrompt);
     }
 
 #if (PYTHON_DISABLE == 0)
@@ -82,7 +82,7 @@ absl::Status ChatTemplateProcessor::process(InputRequest& req) {
         req.promptText = std::move(promptText);
     } else {
 #endif
-        constexpr bool addGenerationPrompt = true;
+        const bool addGenerationPrompt = req.addGenerationPrompt;
         const auto& tools = chatHistory.get_tools();
         const auto& kwargs = chatHistory.get_extra_context();
         const std::optional<ov::genai::JsonContainer> optTools =
diff --git a/src/llm/io_processing/input_request.hpp b/src/llm/io_processing/input_request.hpp
index af382ee400..4bbf1ca1bd 100644
--- a/src/llm/io_processing/input_request.hpp
+++ b/src/llm/io_processing/input_request.hpp
@@ -37,6 +37,7 @@ using InputPayload = std::variant<ov::genai::ChatHistory, std::string>;
 struct InputRequest {
     InputPayload input;                            // set in parseRequest()
     ov::genai::GenerationConfig generationConfig;  // set in parseRequest()
+    bool addGenerationPrompt = true;               // set in parseRequest(); from the request's add_generation_prompt (default true)
 
     std::string promptText;               // written by ChatTemplateProcessor / RawPromptExtractor
     ov::Tensor inputIds;                  // written by TokenizationProcessor (all paths)
diff --git a/src/llm/py_jinja_template_processor.cpp b/src/llm/py_jinja_template_processor.cpp
index 235e374ea8..c3f59ad227 100644
--- a/src/llm/py_jinja_template_processor.cpp
+++ b/src/llm/py_jinja_template_processor.cpp
@@ -58,11 +58,15 @@ bool PyJinjaTemplateProcessor::applyChatTemplate(PyJinjaTemplateProcessor& templ
                 elif not isinstance(chat_template_kwargs, dict):
                     raise Exception("chat_template_kwargs must be an object")
 
+                add_generation_prompt = request_json.get("add_generation_prompt", True)
+                if not isinstance(add_generation_prompt, bool):
+                    raise Exception("add_generation_prompt accepts values true or false")
+
                 tools = request_json["tools"] if "tools" in request_json else None
                 if tools is None:
-                    output = chat_template.render(messages=messages, bos_token=bos_token, eos_token=eos_token, add_generation_prompt=True, **chat_template_kwargs)
+                    output = chat_template.render(messages=messages, bos_token=bos_token, eos_token=eos_token, add_generation_prompt=add_generation_prompt, **chat_template_kwargs)
                 else:
-                    output = tool_chat_template.render(messages=messages, tools=tools, bos_token=bos_token, eos_token=eos_token, add_generation_prompt=True, **chat_template_kwargs)
+                    output = tool_chat_template.render(messages=messages, tools=tools, bos_token=bos_token, eos_token=eos_token, add_generation_prompt=add_generation_prompt, **chat_template_kwargs)
             except Exception as e:
                 error = str(e) 
         )",
diff --git a/src/test/llm/llmtemplate_test.cpp b/src/test/llm/llmtemplate_test.cpp
index e51833cb3b..839f2e9884 100644
--- a/src/test/llm/llmtemplate_test.cpp
+++ b/src/test/llm/llmtemplate_test.cpp
@@ -167,6 +167,38 @@ TEST_F(LLMChatTemplateTest, ChatTemplateDefault) {
     ASSERT_EQ(finalPrompt, expectedOutput);
 }
 
+// add_generation_prompt request field controls whether the trailing generation
+// prompt is rendered (assistant prefill support, issue #3877).
+TEST_F(LLMChatTemplateTest, ChatTemplateAddGenerationPromptDefaultsTrue) {
+    std::string jinja = "{% for message in messages %}{{ message['role'] }}: {{ message['content'] }}{% endfor %}{% if add_generation_prompt %}<|GEN|>{% endif %}";
+    ASSERT_TRUE(CreateJinjaConfig(jinja));
+    LoadTemplateProcessor();
+    std::string finalPrompt = "";
+    std::string payloadBody = R"(
+        {
+            "messages": [{ "role": "user", "content": "hi" }]
+        }
+    )";
+    ASSERT_EQ(PyJinjaTemplateProcessor::applyChatTemplate(servable->getProperties()->templateProcessor, servable->getProperties()->modelsPath, payloadBody, finalPrompt), true);
+    ASSERT_NE(finalPrompt.find("<|GEN|>"), std::string::npos) << "default should add generation prompt, got: " << finalPrompt;
+}
+
+TEST_F(LLMChatTemplateTest, ChatTemplateAddGenerationPromptFalse) {
+    std::string jinja = "{% for message in messages %}{{ message['role'] }}: {{ message['content'] }}{% endfor %}{% if add_generation_prompt %}<|GEN|>{% endif %}";
+    ASSERT_TRUE(CreateJinjaConfig(jinja));
+    LoadTemplateProcessor();
+    std::string finalPrompt = "";
+    std::string payloadBody = R"(
+        {
+            "messages": [{ "role": "user", "content": "hi" }, { "role": "assistant", "content": "partial" }],
+            "add_generation_prompt": false
+        }
+    )";
+    ASSERT_EQ(PyJinjaTemplateProcessor::applyChatTemplate(servable->getProperties()->templateProcessor, servable->getProperties()->modelsPath, payloadBody, finalPrompt), true);
+    ASSERT_EQ(finalPrompt.find("<|GEN|>"), std::string::npos) << "add_generation_prompt=false should omit generation prompt, got: " << finalPrompt;
+    ASSERT_NE(finalPrompt.find("partial"), std::string::npos) << "assistant prefill content should be present, got: " << finalPrompt;
+}
+
 TEST_F(LLMChatTemplateTest, ChatTemplateMultiMessage) {
     CopyDefaultChatTemplate();
     LoadTemplateProcessor();

From de8b3301eebcf452e3cc5dc8415b810c100991ad Mon Sep 17 00:00:00 2001
From: exzile <joeypongallo@gmail.com>
Date: Mon, 29 Jun 2026 10:53:49 -0400
Subject: [PATCH 2/5] Document add_generation_prompt for the responses endpoint

Per review: add_generation_prompt is valid and functional for the
Responses API - the Endpoint::RESPONSES path in servable.cpp reads
request.addGenerationPrompt and passes it to apply_chat_template (and
the Jinja path reads it from the request JSON), same as chat
completions. Document it in the responses REST API parameters table.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 docs/model_server_rest_api_responses.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/model_server_rest_api_responses.md b/docs/model_server_rest_api_responses.md
index b097ca69c7..415b9b77e3 100644
--- a/docs/model_server_rest_api_responses.md
+++ b/docs/model_server_rest_api_responses.md
@@ -106,6 +106,7 @@ curl http://localhost/v3/responses \
 | reasoning | ⚠️ | ✅ | object (optional) | Configuration for reasoning/thinking mode. The `effort` field accepts `"low"`, `"medium"`, or `"high"` — any value enables thinking mode (`enable_thinking: true` is injected into chat template kwargs). The `summary` field is accepted but ignored. |
 | chat_template_kwargs | ✅ | ❌ | object (optional) | Additional keyword arguments passed to the chat template. When `reasoning` is also provided, `enable_thinking: true` is merged into these kwargs. |
 | skip_special_tokens | ✅ | ❌ | bool (default: `true`) | Whether to remove special tokens (e.g. `<\|endoftext\|>`, `<\|im_end\|>`) from the generated output. Set to `false` to include them, which is useful when the model uses special tokens to encode structured information (e.g. bounding boxes, reasoning markers). When `false`, any tool or reasoning parser configured on the endpoint is silently disabled for the request, so the raw token stream is returned. This option works with most detokenizers exported with OpenVINO Tokenizers 2024.5 or later, unless they are based on custom ops. |
+| add_generation_prompt | ✅ | ❌ | bool (default: `true`) | Whether to append the chat template's generation prompt (the marker that signals the model to start a new assistant turn). Set to `false` to render the conversation without a trailing generation prompt — useful for assistant prefill where the final `assistant` message should be continued rather than treated as a completed turn. Applies to both the Python-Jinja and MINJA chat template paths. |
 | stream_options | ❌ | ❌ | | Not supported in Responses API. Usage statistics are always included in the `response.completed` event. |
 
 #### Beam search sampling specific

From a587952c6cf7bd27526fa67bb211270205289e05 Mon Sep 17 00:00:00 2001
From: exzile <joeypongallo@gmail.com>
Date: Thu, 2 Jul 2026 11:35:17 -0400
Subject: [PATCH 3/5] Fold add_generation_prompt into chat_template_kwargs

Per review feedback: add_generation_prompt is only ever consumed by
chat template rendering, so route it through chat_template_kwargs
instead of a dedicated InputRequest field. The MINJA path extracts it
back out for genai's dedicated apply_chat_template argument; the
Python-Jinja path pops it from the kwargs dict before splatting into
render() to avoid a duplicate-keyword collision. Also fixes the
Python-Jinja path never having wired add_generation_prompt through at
all, and two llmtemplate_test.cpp tests that referenced a stale 4-arg
applyChatTemplate signature and never compiled.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 src/llm/apis/openai_api_handler.cpp                  | 10 ++++++----
 .../input_processors/chat_template_processor.cpp     | 12 +++++++++---
 src/llm/io_processing/input_request.hpp              |  3 ++-
 src/llm/py_jinja_template_processor.cpp              |  4 +++-
 src/test/llm/llmtemplate_test.cpp                    |  8 +++++---
 5 files changed, 25 insertions(+), 12 deletions(-)

diff --git a/src/llm/apis/openai_api_handler.cpp b/src/llm/apis/openai_api_handler.cpp
index 15dd44895c..91c8d334a4 100644
--- a/src/llm/apis/openai_api_handler.cpp
+++ b/src/llm/apis/openai_api_handler.cpp
@@ -498,7 +498,6 @@ absl::StatusOr<InputRequest> OpenAIApiHandler::extractInputRequest(GenerationCon
     }
     InputRequest req;
     req.generationConfig = configBuilder.getConfig();
-    req.addGenerationPrompt = request.addGenerationPrompt.value_or(true);
     if (endpoint == Endpoint::COMPLETIONS) {
         req.input = request.prompt.value_or("");
     } else {
@@ -519,9 +518,12 @@ absl::StatusOr<InputRequest> OpenAIApiHandler::extractInputRequest(GenerationCon
         if (!kwargsResult.ok()) {
             return kwargsResult.status();
         }
-        if (kwargsResult.value().has_value()) {
-            chatHistory.set_extra_context(kwargsResult.value().value());
-        }
+        // add_generation_prompt is only ever consumed by chat template rendering, so it is
+        // folded into chat_template_kwargs rather than kept as a separate field, and both the
+        // MINJA and Python-Jinja rendering paths read it from this single source.
+        ov::genai::JsonContainer kwargs = kwargsResult.value().value_or(ov::genai::JsonContainer::object());
+        kwargs["add_generation_prompt"] = request.addGenerationPrompt.value_or(true);
+        chatHistory.set_extra_context(kwargs);
     }
     return req;
 }
diff --git a/src/llm/io_processing/input_processors/chat_template_processor.cpp b/src/llm/io_processing/input_processors/chat_template_processor.cpp
index 619570a795..b076e84dd9 100644
--- a/src/llm/io_processing/input_processors/chat_template_processor.cpp
+++ b/src/llm/io_processing/input_processors/chat_template_processor.cpp
@@ -67,7 +67,6 @@ absl::Status ChatTemplateProcessor::process(InputRequest& req) {
         SPDLOG_LOGGER_TRACE(llm_calculator_logger, "chatHistory.get_extra_context(): {}", chatHistory.get_extra_context().to_json_string());
         SPDLOG_LOGGER_TRACE(llm_calculator_logger, "tools: {}", chatHistory.get_tools().empty() ? std::string("<none>") : chatHistory.get_tools().to_json_string());
         SPDLOG_LOGGER_TRACE(llm_calculator_logger, "chatTemplateKwargs: {}", chatHistory.get_extra_context().empty() ? std::string("<none>") : chatHistory.get_extra_context().to_json_string());
-        SPDLOG_LOGGER_TRACE(llm_calculator_logger, "addGenerationPrompt: {}", req.addGenerationPrompt);
     }
 
 #if (PYTHON_DISABLE == 0)
@@ -82,9 +81,16 @@ absl::Status ChatTemplateProcessor::process(InputRequest& req) {
         req.promptText = std::move(promptText);
     } else {
 #endif
-        const bool addGenerationPrompt = req.addGenerationPrompt;
         const auto& tools = chatHistory.get_tools();
-        const auto& kwargs = chatHistory.get_extra_context();
+        // add_generation_prompt lives inside chat_template_kwargs; MINJA's apply_chat_template
+        // takes it as a dedicated argument, so extract it here and drop it from the kwargs map
+        // passed alongside so it isn't supplied twice.
+        ov::genai::JsonContainer kwargs = chatHistory.get_extra_context();
+        bool addGenerationPrompt = true;
+        if (kwargs.contains("add_generation_prompt")) {
+            addGenerationPrompt = kwargs["add_generation_prompt"].get_bool();
+            kwargs.erase("add_generation_prompt");
+        }
         const std::optional<ov::genai::JsonContainer> optTools =
             tools.empty() ? std::nullopt : std::make_optional(tools);
         const std::optional<ov::genai::JsonContainer> optKwargs =
diff --git a/src/llm/io_processing/input_request.hpp b/src/llm/io_processing/input_request.hpp
index 4bbf1ca1bd..d262c135c4 100644
--- a/src/llm/io_processing/input_request.hpp
+++ b/src/llm/io_processing/input_request.hpp
@@ -37,7 +37,8 @@ using InputPayload = std::variant<ov::genai::ChatHistory, std::string>;
 struct InputRequest {
     InputPayload input;                            // set in parseRequest()
     ov::genai::GenerationConfig generationConfig;  // set in parseRequest()
-    bool addGenerationPrompt = true;               // set in parseRequest(); from the request's add_generation_prompt (default true)
+    // add_generation_prompt is folded into ChatHistory's extra_context (chat_template_kwargs)
+    // by extractInputRequest() since it is only ever consumed by chat template rendering.
 
     std::string promptText;               // written by ChatTemplateProcessor / RawPromptExtractor
     ov::Tensor inputIds;                  // written by TokenizationProcessor (all paths)
diff --git a/src/llm/py_jinja_template_processor.cpp b/src/llm/py_jinja_template_processor.cpp
index c3f59ad227..abd4c16de8 100644
--- a/src/llm/py_jinja_template_processor.cpp
+++ b/src/llm/py_jinja_template_processor.cpp
@@ -58,7 +58,9 @@ bool PyJinjaTemplateProcessor::applyChatTemplate(PyJinjaTemplateProcessor& templ
                 elif not isinstance(chat_template_kwargs, dict):
                     raise Exception("chat_template_kwargs must be an object")
 
-                add_generation_prompt = request_json.get("add_generation_prompt", True)
+                # add_generation_prompt is passed as part of chat_template_kwargs; pop it out so
+                # it is not also supplied via **chat_template_kwargs below (duplicate keyword).
+                add_generation_prompt = chat_template_kwargs.pop("add_generation_prompt", True)
                 if not isinstance(add_generation_prompt, bool):
                     raise Exception("add_generation_prompt accepts values true or false")
 
diff --git a/src/test/llm/llmtemplate_test.cpp b/src/test/llm/llmtemplate_test.cpp
index 839f2e9884..15168d49cb 100644
--- a/src/test/llm/llmtemplate_test.cpp
+++ b/src/test/llm/llmtemplate_test.cpp
@@ -179,7 +179,7 @@ TEST_F(LLMChatTemplateTest, ChatTemplateAddGenerationPromptDefaultsTrue) {
             "messages": [{ "role": "user", "content": "hi" }]
         }
     )";
-    ASSERT_EQ(PyJinjaTemplateProcessor::applyChatTemplate(servable->getProperties()->templateProcessor, servable->getProperties()->modelsPath, payloadBody, finalPrompt), true);
+    ASSERT_EQ(PyJinjaTemplateProcessor::applyChatTemplate(servable->getProperties()->templateProcessor, payloadBody, finalPrompt), true);
     ASSERT_NE(finalPrompt.find("<|GEN|>"), std::string::npos) << "default should add generation prompt, got: " << finalPrompt;
 }
 
@@ -188,13 +188,15 @@ TEST_F(LLMChatTemplateTest, ChatTemplateAddGenerationPromptFalse) {
     ASSERT_TRUE(CreateJinjaConfig(jinja));
     LoadTemplateProcessor();
     std::string finalPrompt = "";
+    // add_generation_prompt is folded into chat_template_kwargs by the OpenAI API handler
+    // before applyChatTemplate is ever called; reflect that contract here.
     std::string payloadBody = R"(
         {
             "messages": [{ "role": "user", "content": "hi" }, { "role": "assistant", "content": "partial" }],
-            "add_generation_prompt": false
+            "chat_template_kwargs": { "add_generation_prompt": false }
         }
     )";
-    ASSERT_EQ(PyJinjaTemplateProcessor::applyChatTemplate(servable->getProperties()->templateProcessor, servable->getProperties()->modelsPath, payloadBody, finalPrompt), true);
+    ASSERT_EQ(PyJinjaTemplateProcessor::applyChatTemplate(servable->getProperties()->templateProcessor, payloadBody, finalPrompt), true);
     ASSERT_EQ(finalPrompt.find("<|GEN|>"), std::string::npos) << "add_generation_prompt=false should omit generation prompt, got: " << finalPrompt;
     ASSERT_NE(finalPrompt.find("partial"), std::string::npos) << "assistant prefill content should be present, got: " << finalPrompt;
 }

From cb0043681879d3203245de725d0667832d527ea0 Mon Sep 17 00:00:00 2001
From: exzile <joeypongallo@gmail.com>
Date: Fri, 3 Jul 2026 12:25:30 -0400
Subject: [PATCH 4/5] Address review: drop leftover top-level
 add_generation_prompt field, validate kwargs type

- Remove the now-redundant top-level add_generation_prompt request field/parsing
  and the OpenAIRequest.addGenerationPrompt member: the parameter lives entirely
  in chat_template_kwargs now, and both template paths already default it to
  true when absent.
- Fix MINJA path to use JsonContainer::as_bool() instead of get_bool(), which
  threw an unhandled ov::Exception (outside the surrounding try/catch) when
  add_generation_prompt was present but not a boolean; now returns a clean
  InvalidArgumentError, matching the Python-Jinja path's existing validation.
- Trim a leftover comment line in input_request.hpp per review suggestion.
- Update chat/responses REST docs to document add_generation_prompt as part of
  chat_template_kwargs instead of a separate top-level parameter.
- Add unit tests for add_generation_prompt=false and the invalid-type rejection
  on the MINJA path.
---
 docs/model_server_rest_api_chat.md            |  3 +-
 docs/model_server_rest_api_responses.md       |  3 +-
 src/llm/apis/openai_api_handler.cpp           | 21 +++--------
 src/llm/apis/openai_request.hpp               |  3 --
 .../chat_template_processor.cpp               |  7 +++-
 src/llm/io_processing/input_request.hpp       |  3 +-
 .../chat_template_processor_test.cpp          | 37 +++++++++++++++++++
 7 files changed, 51 insertions(+), 26 deletions(-)

diff --git a/docs/model_server_rest_api_chat.md b/docs/model_server_rest_api_chat.md
index 332fb330a2..2888d61dc0 100644
--- a/docs/model_server_rest_api_chat.md
+++ b/docs/model_server_rest_api_chat.md
@@ -220,9 +220,8 @@ Some parameters, especially related to sampling (like `temperature`, `top_p` etc
 | tools | ✅ | ✅ | ✅ | array | A list of tools the model may call. Currently, only functions are supported as a tool. Use this to provide a list of functions the model may generate JSON inputs for. See [OpenAI API reference](https://platform.openai.com/docs/api-reference/chat/create#chat-create-tools) for more details. |
 | tool_choice | ✅ | ✅ | ✅ | string or object | Controls which (if any) tool is called by the model. `none` means the model will not call any tool and instead generates a message. `auto` means the model can pick between generating a message or calling one or more tools. `required` means that model should call at least one tool. Specifying a particular tool via `{"type": "function", "function": {"name": "my_function"}}` forces the model to call that tool. See [OpenAI API reference](https://platform.openai.com/docs/api-reference/chat/create#chat-create-tool_choice) for more details. |
 | response_format | ✅ | ✅ | ✅ | object | An object specifying the format that the model must output. Setting to `{ "type": "json_schema", "json_schema": {...} }` enables Structured Outputs which ensures the model will match your supplied JSON schema according to [OpenAI reference](https://platform.openai.com/docs/api-reference/chat/create#chat-create-response_format). Learn more in the [Structured Outputs demo](../demos/continuous_batching/structured_output/README.md). Additionally, `response_format` can accept [XGrammar structural tags format](https://github.com/mlc-ai/xgrammar/blob/v0.1.26/docs/tutorials/structural_tag.md#format-types) (not part of OpenAI API). For example: `{ "type": "const_string", "value": "Hello World!" }`. **Note** that if model server fails to process the format, the request will still be processed, but the format will not be imposed. |
-| chat_template_kwargs | ✅ | ❌ | ✅ |  object | Enables passing additional parameters to chat template engine. Example `{"enable_thinking": false}`. Note that values like `messages`, `eos_token`, `bos_token` etc. are provided natively to the template engine, so including them in `chat_template_kwargs` will cause error. |
+| chat_template_kwargs | ✅ | ❌ | ✅ |  object | Enables passing additional parameters to chat template engine. Example `{"enable_thinking": false}`. Note that values like `messages`, `eos_token`, `bos_token` etc. are provided natively to the template engine, so including them in `chat_template_kwargs` will cause error. Also accepts `add_generation_prompt` (bool, default: `true`) — whether to append the chat template's generation prompt (the marker that signals the model to start a new assistant turn). Set to `false` to render the conversation without a trailing generation prompt, e.g. `{"add_generation_prompt": false}` — useful for assistant prefill where the final `assistant` message should be continued rather than treated as a completed turn. Applies to both the Python-Jinja and MINJA chat template paths. |
 | skip_special_tokens | ✅ | ❌ | ✅ | bool (default: `true`) | Whether to remove special tokens (e.g. `<\|endoftext\|>`, `<\|im_end\|>`) from the generated output. Set to `false` to include them, which is useful when the model uses special tokens to encode structured information (e.g. bounding boxes, reasoning markers). When `false`, any tool or reasoning parser configured on the endpoint is silently disabled for the request, so the raw token stream is returned. This option works with most detokenizers exported with OpenVINO Tokenizers 2024.5 or later, unless they are based on custom ops. |
-| add_generation_prompt | ✅ | ❌ | ✅ | bool (default: `true`) | Whether to append the chat template's generation prompt (the marker that signals the model to start a new assistant turn). Set to `false` to render the conversation without a trailing generation prompt — useful for assistant prefill where the final `assistant` message should be continued rather than treated as a completed turn. Applies to both the Python-Jinja and MINJA chat template paths. |
 
 #### Beam search sampling specific
 | Param | OpenVINO Model Server | OpenAI /chat/completions API | vLLM Serving Sampling Params | Type | Description |
diff --git a/docs/model_server_rest_api_responses.md b/docs/model_server_rest_api_responses.md
index 415b9b77e3..2ff2f2c246 100644
--- a/docs/model_server_rest_api_responses.md
+++ b/docs/model_server_rest_api_responses.md
@@ -104,9 +104,8 @@ curl http://localhost/v3/responses \
 | tools | ⚠️ | ✅ | array (optional) | A list of tools the model may call. Currently, only **function** tools are supported. OpenAI also supports built-in tools (web_search, file_search, code_interpreter, etc.) and MCP tools. OVMS additionally accepts a flat `{type, name, parameters}` format alongside the nested `{type, function: {name, parameters}}` format. See [OpenAI API reference](https://platform.openai.com/docs/api-reference/chat/create#chat-create-tools) for more details. |
 | tool_choice | ✅ | ✅ | string or object (optional) | Controls which (if any) tool is called by the model. `none` means the model will not call any tool and instead generates a message. `auto` means the model can pick between generating a message or calling one or more tools. `required` means that model should call at least one tool. Specifying a particular function via `{"type": "function", "function": {"name": "my_function"}}` forces the model to call that tool. |
 | reasoning | ⚠️ | ✅ | object (optional) | Configuration for reasoning/thinking mode. The `effort` field accepts `"low"`, `"medium"`, or `"high"` — any value enables thinking mode (`enable_thinking: true` is injected into chat template kwargs). The `summary` field is accepted but ignored. |
-| chat_template_kwargs | ✅ | ❌ | object (optional) | Additional keyword arguments passed to the chat template. When `reasoning` is also provided, `enable_thinking: true` is merged into these kwargs. |
+| chat_template_kwargs | ✅ | ❌ | object (optional) | Additional keyword arguments passed to the chat template. When `reasoning` is also provided, `enable_thinking: true` is merged into these kwargs. Also accepts `add_generation_prompt` (bool, default: `true`) — whether to append the chat template's generation prompt (the marker that signals the model to start a new assistant turn). Set to `false` to render the conversation without a trailing generation prompt, e.g. `{"add_generation_prompt": false}` — useful for assistant prefill where the final `assistant` message should be continued rather than treated as a completed turn. Applies to both the Python-Jinja and MINJA chat template paths. |
 | skip_special_tokens | ✅ | ❌ | bool (default: `true`) | Whether to remove special tokens (e.g. `<\|endoftext\|>`, `<\|im_end\|>`) from the generated output. Set to `false` to include them, which is useful when the model uses special tokens to encode structured information (e.g. bounding boxes, reasoning markers). When `false`, any tool or reasoning parser configured on the endpoint is silently disabled for the request, so the raw token stream is returned. This option works with most detokenizers exported with OpenVINO Tokenizers 2024.5 or later, unless they are based on custom ops. |
-| add_generation_prompt | ✅ | ❌ | bool (default: `true`) | Whether to append the chat template's generation prompt (the marker that signals the model to start a new assistant turn). Set to `false` to render the conversation without a trailing generation prompt — useful for assistant prefill where the final `assistant` message should be continued rather than treated as a completed turn. Applies to both the Python-Jinja and MINJA chat template paths. |
 | stream_options | ❌ | ❌ | | Not supported in Responses API. Usage statistics are always included in the `response.completed` event. |
 
 #### Beam search sampling specific
diff --git a/src/llm/apis/openai_api_handler.cpp b/src/llm/apis/openai_api_handler.cpp
index 644c42273b..7a66ec3246 100644
--- a/src/llm/apis/openai_api_handler.cpp
+++ b/src/llm/apis/openai_api_handler.cpp
@@ -359,11 +359,11 @@ absl::StatusOr<InputRequest> OpenAIApiHandler::extractInputRequest(GenerationCon
             return kwargsResult.status();
         }
         // add_generation_prompt is only ever consumed by chat template rendering, so it is
-        // folded into chat_template_kwargs rather than kept as a separate field, and both the
-        // MINJA and Python-Jinja rendering paths read it from this single source.
-        ov::genai::JsonContainer kwargs = kwargsResult.value().value_or(ov::genai::JsonContainer::object());
-        kwargs["add_generation_prompt"] = request.addGenerationPrompt.value_or(true);
-        chatHistory.set_extra_context(kwargs);
+        // read directly out of chat_template_kwargs (both the MINJA and Python-Jinja rendering
+        // paths default it to true when absent) rather than kept as a separate request field.
+        if (kwargsResult.value().has_value()) {
+            chatHistory.set_extra_context(kwargsResult.value().value());
+        }
     }
     return req;
 }
@@ -528,17 +528,6 @@ absl::Status OpenAIApiHandler::parseCommonPart(std::optional<uint32_t> maxTokens
         request.ignoreEOS = it->value.GetBool();
     }
 
-    // add_generation_prompt: bool; optional - defaults to true
-    // Extension, unsupported by OpenAI API, however supported by HF transformers and vLLM.
-    // When false, the chat template is rendered without a trailing generation prompt
-    // so a final assistant message can be continued as a prefix (assistant prefill).
-    it = doc.FindMember("add_generation_prompt");
-    if (it != doc.MemberEnd() && !it->value.IsNull()) {
-        if (!it->value.IsBool())
-            return absl::InvalidArgumentError("add_generation_prompt accepts values true or false");
-        request.addGenerationPrompt = it->value.GetBool();
-    }
-
     // max_tokens: uint; optional
     // Common part checked here, specific parts are checked in parseCompletionsPart and parseChatCompletionsPart
     // TODO: Deprecated - this will need to be removed in the future
diff --git a/src/llm/apis/openai_request.hpp b/src/llm/apis/openai_request.hpp
index faf100f94a..5b433a7a0b 100644
--- a/src/llm/apis/openai_request.hpp
+++ b/src/llm/apis/openai_request.hpp
@@ -48,9 +48,6 @@ struct OpenAIRequest {
     int logprobschat{0};
     bool echo{false};
     std::optional<bool> ignoreEOS{std::nullopt};
-    // When false, the chat template is rendered without a trailing generation prompt
-    // (e.g. for assistant prefill). Defaults to true. Extension supported by HF/vLLM.
-    std::optional<bool> addGenerationPrompt{std::nullopt};
     std::optional<std::set<std::string>> stop{std::nullopt};
     std::optional<bool> includeStopStrInOutput{std::nullopt};
     std::optional<int> numReturnSequences{std::nullopt};  // effective for beam search and multinomial decoding
diff --git a/src/llm/io_processing/input_processors/chat_template_processor.cpp b/src/llm/io_processing/input_processors/chat_template_processor.cpp
index b076e84dd9..25373b451d 100644
--- a/src/llm/io_processing/input_processors/chat_template_processor.cpp
+++ b/src/llm/io_processing/input_processors/chat_template_processor.cpp
@@ -88,7 +88,12 @@ absl::Status ChatTemplateProcessor::process(InputRequest& req) {
         ov::genai::JsonContainer kwargs = chatHistory.get_extra_context();
         bool addGenerationPrompt = true;
         if (kwargs.contains("add_generation_prompt")) {
-            addGenerationPrompt = kwargs["add_generation_prompt"].get_bool();
+            const auto asBool = kwargs["add_generation_prompt"].as_bool();
+            if (!asBool.has_value()) {
+                return absl::Status(absl::StatusCode::kInvalidArgument,
+                    "add_generation_prompt accepts values true or false");
+            }
+            addGenerationPrompt = asBool.value();
             kwargs.erase("add_generation_prompt");
         }
         const std::optional<ov::genai::JsonContainer> optTools =
diff --git a/src/llm/io_processing/input_request.hpp b/src/llm/io_processing/input_request.hpp
index d262c135c4..ec2fce300f 100644
--- a/src/llm/io_processing/input_request.hpp
+++ b/src/llm/io_processing/input_request.hpp
@@ -37,8 +37,7 @@ using InputPayload = std::variant<ov::genai::ChatHistory, std::string>;
 struct InputRequest {
     InputPayload input;                            // set in parseRequest()
     ov::genai::GenerationConfig generationConfig;  // set in parseRequest()
-    // add_generation_prompt is folded into ChatHistory's extra_context (chat_template_kwargs)
-    // by extractInputRequest() since it is only ever consumed by chat template rendering.
+    // add_generation_prompt is folded into ChatHistory's extra_context (chat_template_kwargs).
 
     std::string promptText;               // written by ChatTemplateProcessor / RawPromptExtractor
     ov::Tensor inputIds;                  // written by TokenizationProcessor (all paths)
diff --git a/src/test/llm/input_processing/chat_template_processor_test.cpp b/src/test/llm/input_processing/chat_template_processor_test.cpp
index e61060173e..1cbee1ae65 100644
--- a/src/test/llm/input_processing/chat_template_processor_test.cpp
+++ b/src/test/llm/input_processing/chat_template_processor_test.cpp
@@ -141,6 +141,43 @@ TEST_F(ChatTemplateProcessorTest, MultiTurnConversation_AllTurnsRendered) {
     EXPECT_EQ(req.promptText, expected);
 }
 
+// add_generation_prompt=false (read out of chat_template_kwargs) must omit the
+// trailing generation-prompt suffix while otherwise rendering normally.
+TEST_F(ChatTemplateProcessorTest, AddGenerationPromptFalse_OmitsGenerationPromptSuffix) {
+    ov::genai::ChatHistory history;
+    history.push_back({{"role", "user"}, {"content", "What is OpenVINO?"}});
+    history.set_extra_context(ov::genai::JsonContainer::from_json_string(R"({"add_generation_prompt": false})"));
+
+    InputRequest req = makeChatRequest(std::move(history));
+    ChatTemplateProcessor processor(*sharedTokenizer);
+    const auto status = processor.process(req);
+
+    ASSERT_TRUE(status.ok()) << status.message();
+
+    const std::string expected =
+        std::string(SMOL_DEFAULT_SYSTEM) +
+        "<|im_start|>user\nWhat is OpenVINO?<|im_end|>\n";
+    EXPECT_EQ(req.promptText, expected);
+    EXPECT_EQ(req.promptText.find("<|im_start|>assistant"), std::string::npos)
+        << "add_generation_prompt=false must omit the trailing generation prompt";
+}
+
+// A non-boolean add_generation_prompt must be rejected with a clear error
+// instead of throwing an unhandled JsonContainer type-mismatch exception.
+TEST_F(ChatTemplateProcessorTest, AddGenerationPromptNonBoolean_ReturnsInvalidArgument) {
+    ov::genai::ChatHistory history;
+    history.push_back({{"role", "user"}, {"content", "Hi."}});
+    history.set_extra_context(ov::genai::JsonContainer::from_json_string(R"({"add_generation_prompt": "yes"})"));
+
+    InputRequest req = makeChatRequest(std::move(history));
+    ChatTemplateProcessor processor(*sharedTokenizer);
+    const auto status = processor.process(req);
+
+    ASSERT_FALSE(status.ok());
+    EXPECT_EQ(status.code(), absl::StatusCode::kInvalidArgument);
+    EXPECT_NE(status.message().find("add_generation_prompt"), std::string::npos) << status.message();
+}
+
 // The processor must populate req.promptText and leave the ChatHistory
 // variant in req.input intact (it does not replace the input variant).
 TEST_F(ChatTemplateProcessorTest, PromptTextPopulated_ChatHistoryVariantPreserved) {

From 3d3b6e5fe4d03fca0dbaac119dc06337921b5f89 Mon Sep 17 00:00:00 2001
From: exzile <joeypongallo@gmail.com>
Date: Tue, 7 Jul 2026 09:30:44 -0400
Subject: [PATCH 5/5] Address further review: drop singled-out comments,
 extract add_generation_prompt parsing

- Remove the comments in openai_api_handler.cpp and input_request.hpp that
  called out add_generation_prompt specifically among chat_template_kwargs
  entries (mzegla) -- the field-level docs already cover it.
- Extract the add_generation_prompt lookup/validation in
  ChatTemplateProcessor::process into its own extractAddGenerationPrompt
  method (dkalinowski).
---
 src/llm/apis/openai_api_handler.cpp           |  3 --
 .../chat_template_processor.cpp               | 32 ++++++++++++-------
 .../chat_template_processor.hpp               |  6 ++++
 src/llm/io_processing/input_request.hpp       |  1 -
 4 files changed, 26 insertions(+), 16 deletions(-)

diff --git a/src/llm/apis/openai_api_handler.cpp b/src/llm/apis/openai_api_handler.cpp
index 7a66ec3246..c7dc3a93cc 100644
--- a/src/llm/apis/openai_api_handler.cpp
+++ b/src/llm/apis/openai_api_handler.cpp
@@ -358,9 +358,6 @@ absl::StatusOr<InputRequest> OpenAIApiHandler::extractInputRequest(GenerationCon
         if (!kwargsResult.ok()) {
             return kwargsResult.status();
         }
-        // add_generation_prompt is only ever consumed by chat template rendering, so it is
-        // read directly out of chat_template_kwargs (both the MINJA and Python-Jinja rendering
-        // paths default it to true when absent) rather than kept as a separate request field.
         if (kwargsResult.value().has_value()) {
             chatHistory.set_extra_context(kwargsResult.value().value());
         }
diff --git a/src/llm/io_processing/input_processors/chat_template_processor.cpp b/src/llm/io_processing/input_processors/chat_template_processor.cpp
index 25373b451d..83a04b3837 100644
--- a/src/llm/io_processing/input_processors/chat_template_processor.cpp
+++ b/src/llm/io_processing/input_processors/chat_template_processor.cpp
@@ -56,6 +56,22 @@ ChatTemplateProcessor::ChatTemplateProcessor(ov::genai::Tokenizer& tokenizer) :
     tokenizer(tokenizer) {}
 #endif
 
+absl::Status ChatTemplateProcessor::extractAddGenerationPrompt(const ov::genai::ChatHistory& chatHistory,
+    ov::genai::JsonContainer& kwargs, bool& addGenerationPrompt) {
+    kwargs = chatHistory.get_extra_context();
+    addGenerationPrompt = true;
+    if (kwargs.contains("add_generation_prompt")) {
+        const auto asBool = kwargs["add_generation_prompt"].as_bool();
+        if (!asBool.has_value()) {
+            return absl::Status(absl::StatusCode::kInvalidArgument,
+                "add_generation_prompt accepts values true or false");
+        }
+        addGenerationPrompt = asBool.value();
+        kwargs.erase("add_generation_prompt");
+    }
+    return absl::OkStatus();
+}
+
 absl::Status ChatTemplateProcessor::process(InputRequest& req) {
     if (!std::holds_alternative<ov::genai::ChatHistory>(req.input)) {
         return absl::Status(absl::StatusCode::kInternal,
@@ -82,19 +98,11 @@ absl::Status ChatTemplateProcessor::process(InputRequest& req) {
     } else {
 #endif
         const auto& tools = chatHistory.get_tools();
-        // add_generation_prompt lives inside chat_template_kwargs; MINJA's apply_chat_template
-        // takes it as a dedicated argument, so extract it here and drop it from the kwargs map
-        // passed alongside so it isn't supplied twice.
-        ov::genai::JsonContainer kwargs = chatHistory.get_extra_context();
+        ov::genai::JsonContainer kwargs;
         bool addGenerationPrompt = true;
-        if (kwargs.contains("add_generation_prompt")) {
-            const auto asBool = kwargs["add_generation_prompt"].as_bool();
-            if (!asBool.has_value()) {
-                return absl::Status(absl::StatusCode::kInvalidArgument,
-                    "add_generation_prompt accepts values true or false");
-            }
-            addGenerationPrompt = asBool.value();
-            kwargs.erase("add_generation_prompt");
+        auto status = extractAddGenerationPrompt(chatHistory, kwargs, addGenerationPrompt);
+        if (!status.ok()) {
+            return status;
         }
         const std::optional<ov::genai::JsonContainer> optTools =
             tools.empty() ? std::nullopt : std::make_optional(tools);
diff --git a/src/llm/io_processing/input_processors/chat_template_processor.hpp b/src/llm/io_processing/input_processors/chat_template_processor.hpp
index 1f3a0a0ee8..108a593887 100644
--- a/src/llm/io_processing/input_processors/chat_template_processor.hpp
+++ b/src/llm/io_processing/input_processors/chat_template_processor.hpp
@@ -52,6 +52,12 @@ class ChatTemplateProcessor : public BaseInputProcessor {
 
 private:
     ov::genai::Tokenizer& tokenizer;  // non-owning; lifetime tied to InputProcessorContext
+
+    // add_generation_prompt lives inside chat_template_kwargs; MINJA's apply_chat_template
+    // takes it as a dedicated argument, so this extracts it out and drops it from the returned
+    // kwargs so it isn't supplied twice.
+    static absl::Status extractAddGenerationPrompt(const ov::genai::ChatHistory& chatHistory,
+        ov::genai::JsonContainer& kwargs, bool& addGenerationPrompt);
 #if (PYTHON_DISABLE == 0)
     // Present only on the PyJinja path; nullopt → use tokenizer.apply_chat_template().
     std::optional<std::reference_wrapper<PyJinjaTemplateProcessor>> templateProcessor;
diff --git a/src/llm/io_processing/input_request.hpp b/src/llm/io_processing/input_request.hpp
index ec2fce300f..af382ee400 100644
--- a/src/llm/io_processing/input_request.hpp
+++ b/src/llm/io_processing/input_request.hpp
@@ -37,7 +37,6 @@ using InputPayload = std::variant<ov::genai::ChatHistory, std::string>;
 struct InputRequest {
     InputPayload input;                            // set in parseRequest()
     ov::genai::GenerationConfig generationConfig;  // set in parseRequest()
-    // add_generation_prompt is folded into ChatHistory's extra_context (chat_template_kwargs).
 
     std::string promptText;               // written by ChatTemplateProcessor / RawPromptExtractor
     ov::Tensor inputIds;                  // written by TokenizationProcessor (all paths)