From ea8f2b64816a152c8db28c2b8a9f9ff5a9994fe0 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 8 Aug 2026 14:33:36 +0300 Subject: [PATCH 1/4] feat(providers): first-class LM Studio local-runtime preset Co-authored-by: Medulla --- providers.env.example | 25 ++++-- src/harness/providers/openai/transport.rs | 25 +++++- src/harness/providers/types.rs | 16 ++++ tests/live_provider_matrix.rs | 96 +++++++++++++++++++++-- 4 files changed, 147 insertions(+), 15 deletions(-) diff --git a/providers.env.example b/providers.env.example index 27fcf00..55679e9 100644 --- a/providers.env.example +++ b/providers.env.example @@ -62,8 +62,10 @@ PROVIDER_MISTRAL_PRESET=mistral PROVIDER_MISTRAL_API_KEY= PROVIDER_MISTRAL_MODEL=mistral-small-latest -# Local Ollama needs no real credential — set any non-blank value (for example -# `local`) to opt it into the run; blank keeps it skipped like every other row. +# Local Ollama needs no credential and is dialled with a blank _API_KEY, unlike +# every hosted row — there is no key to withhold, so requiring a placeholder +# would only ever exclude a working local server. Comment the row out (or stop +# the server) to leave it out of a run. PROVIDER_OLLAMA_PRESET=ollama PROVIDER_OLLAMA_API_KEY= # Must match the pulled tag exactly — Ollama 404s on an untagged name it has @@ -134,10 +136,23 @@ PROVIDER_ZHIPU_MODEL=glm-4-flash # --------------------------------------------------------------------------- # Self-hosted / local servers (OpenAI-compatible) # --------------------------------------------------------------------------- - -PROVIDER_LMSTUDIO_BASE_URL=http://localhost:1234/v1 +# +# For a deeper local-runtime check than this matrix — a full tool round trip, +# structured output, and real embeddings + retrieval — see +# `tests/live_local_models.rs` and `tests/live_local_embeddings.rs` +# (`LOCAL_MODEL_TESTS=1 cargo test --test live_local_models`). Those discover +# the served model from the server itself and need no configuration here. + +# LM Studio has a built-in preset, so it needs no _BASE_URL unless the server +# is remote or on a non-default port. The preset sets the local-runtime +# behaviour the plain _BASE_URL form does not: no Authorization header, and the +# request-shape degradations llama.cpp-backed servers require. +PROVIDER_LMSTUDIO_PRESET=lmstudio PROVIDER_LMSTUDIO_API_KEY= -PROVIDER_LMSTUDIO_MODEL=local-model +# REQUIRED — unlike every other preset, LM Studio has no default model: the id +# is whatever GGUF you loaded. List them with `lms ps`, or +# `curl localhost:1234/v1/models`. +PROVIDER_LMSTUDIO_MODEL=qwen/qwen3-4b PROVIDER_VLLM_BASE_URL=http://localhost:8000/v1 PROVIDER_VLLM_API_KEY= diff --git a/src/harness/providers/openai/transport.rs b/src/harness/providers/openai/transport.rs index b35e29f..447905d 100644 --- a/src/harness/providers/openai/transport.rs +++ b/src/harness/providers/openai/transport.rs @@ -733,7 +733,11 @@ impl OpenAiModel { "provider spec base_url must not be empty".to_string(), )); } - if spec.kind == crate::harness::providers::ProviderKind::Ollama { + // Local runtimes need the same treatment whichever one it is: no + // Authorization header, a base URL normalised to the `/v1` root, and + // the request-shape degradations these servers require. Ollama and LM + // Studio differ only in their default port. + if let Some(default_root) = local_runtime_default_root(&spec.kind) { let auth = if spec.requires_api_key { AuthStyle::Bearer } else { @@ -741,7 +745,7 @@ impl OpenAiModel { }; return Ok(Self::local_runtime( &spec.provider, - normalize_local_v1_base_url(spec.base_url, "http://localhost:11434")?, + normalize_local_v1_base_url(spec.base_url, default_root)?, api_key, spec.model, ) @@ -1454,6 +1458,23 @@ impl OpenAiModel { } } +/// The server root a local-runtime provider falls back to when its spec carries +/// a blank `base_url`, or `None` for providers that are not local runtimes. +/// +/// This is the single place that decides "is this kind a local runtime?", so a +/// new local provider is one arm here rather than a condition to keep in sync +/// across the transport. +fn local_runtime_default_root( + kind: &crate::harness::providers::ProviderKind, +) -> Option<&'static str> { + use crate::harness::providers::ProviderKind; + match kind { + ProviderKind::Ollama => Some("http://localhost:11434"), + ProviderKind::LmStudio => Some("http://localhost:1234"), + _ => None, + } +} + fn normalize_local_v1_base_url(raw: String, default_root: &str) -> Result { let trimmed = raw.trim().trim_end_matches('/'); let root = if trimmed.is_empty() { diff --git a/src/harness/providers/types.rs b/src/harness/providers/types.rs index 8c5bb38..c75604b 100644 --- a/src/harness/providers/types.rs +++ b/src/harness/providers/types.rs @@ -28,6 +28,14 @@ pub enum ProviderKind { Anthropic, /// Local Ollama server exposing `/v1/chat/completions`. Ollama, + /// Local LM Studio server exposing `/v1/chat/completions`. + /// + /// Unlike every other preset this one carries **no default model**: the id + /// LM Studio serves is whatever GGUF the user loaded, so there is no + /// stable name to guess. Set one explicitly with + /// [`ProviderSpec::with_model`], or discover it at runtime with + /// [`OpenAiModel::list_models`](crate::harness::providers::openai::OpenAiModel::list_models). + LmStudio, /// DeepSeek OpenAI-compatible endpoint. DeepSeek, /// Groq OpenAI-compatible endpoint. @@ -51,6 +59,7 @@ impl ProviderKind { ProviderKind::OpenAi => "openai", ProviderKind::Anthropic => "anthropic", ProviderKind::Ollama => "ollama", + ProviderKind::LmStudio => "lmstudio", ProviderKind::DeepSeek => "deepseek", ProviderKind::Groq => "groq", ProviderKind::Xai => "xai", @@ -73,6 +82,7 @@ impl ProviderKind { "openai" => Some(ProviderKind::OpenAi), "anthropic" => Some(ProviderKind::Anthropic), "ollama" => Some(ProviderKind::Ollama), + "lmstudio" | "lm_studio" | "lm-studio" => Some(ProviderKind::LmStudio), "deepseek" => Some(ProviderKind::DeepSeek), "groq" => Some(ProviderKind::Groq), "xai" => Some(ProviderKind::Xai), @@ -142,6 +152,12 @@ impl ProviderSpec { ProviderKind::Ollama => { Self::new(kind, "llama3.2", "http://localhost:11434/v1", None, false) } + // No default model on purpose: LM Studio serves whichever GGUF is + // loaded, so any id guessed here would 404 on most installs. An + // empty model forces callers to set one (or discover it through + // `list_models`), which fails loudly at construction instead of + // silently on the first request. + ProviderKind::LmStudio => Self::new(kind, "", "http://localhost:1234/v1", None, false), ProviderKind::DeepSeek => Self::new( kind, "deepseek-chat", diff --git a/tests/live_provider_matrix.rs b/tests/live_provider_matrix.rs index dc72a1c..92c6191 100644 --- a/tests/live_provider_matrix.rs +++ b/tests/live_provider_matrix.rs @@ -167,6 +167,7 @@ fn preset_kind(name: &str) -> Option { "openai" => Some(ProviderKind::OpenAi), "anthropic" => Some(ProviderKind::Anthropic), "ollama" => Some(ProviderKind::Ollama), + "lmstudio" | "lm_studio" | "lm-studio" => Some(ProviderKind::LmStudio), "deepseek" => Some(ProviderKind::DeepSeek), "groq" => Some(ProviderKind::Groq), "xai" => Some(ProviderKind::Xai), @@ -268,6 +269,13 @@ fn resolve(entry: &ProviderEntry, env_lookup: &dyn Fn(&str) -> Option) - .filter(|k| !k.trim().is_empty()) }); + // A local runtime (`requires_api_key: false`) has no credential to find and + // never sends an Authorization header, so a missing key must not skip it. + // Requiring a placeholder — which is what the Ollama row used to document — + // meant a correctly-configured local server was silently excluded from the + // matrix by an empty variable that could never be filled in meaningfully. + let api_key = api_key.or_else(|| (!spec.requires_api_key).then(|| "local".to_string())); + match api_key { Some(api_key) => Resolution::Ready(Box::new(ResolvedProvider { name, @@ -872,13 +880,29 @@ fn every_built_in_preset_name_resolves() { "openrouter", "together", "mistral", + "lmstudio", "ollama", ] { let kind = preset_kind(name).unwrap_or_else(|| panic!("preset {name} should resolve")); - let spec = ProviderSpec::for_kind(kind); + let spec = ProviderSpec::for_kind(kind.clone()); + assert!( + !spec.base_url.is_empty(), + "preset {name} must carry a default base_url" + ); + // LM Studio is the one preset with no default model, and deliberately + // so: the served id is whatever GGUF the operator loaded, so any guess + // would 404 on most installs. It is the only exemption — a new preset + // that cannot name a model belongs behind a base URL instead. + if kind == ProviderKind::LmStudio { + assert!( + spec.model.is_empty(), + "the LM Studio preset must not guess a model id" + ); + continue; + } assert!( - !spec.base_url.is_empty() && !spec.model.is_empty(), - "preset {name} must carry a default base_url and model" + !spec.model.is_empty(), + "preset {name} must carry a default model" ); } assert_eq!(preset_kind("nope"), None); @@ -993,6 +1017,7 @@ fn the_example_file_documents_every_built_in_preset() { "openrouter", "together", "mistral", + "lmstudio", ] { assert!( entries.iter().any(|e| e.preset.as_deref() == Some(preset)), @@ -1009,10 +1034,65 @@ fn the_example_file_documents_every_built_in_preset() { "{} must ship with a blank API key", entry.slug ); - assert!( - matches!(resolve(entry, &|_| None), Resolution::Skipped(_)), - "{} should resolve cleanly and skip on a blank key", - entry.slug - ); + + let resolution = resolve(entry, &|_| None); + let local = entry + .preset + .as_deref() + .and_then(preset_kind) + .is_some_and(|kind| !ProviderSpec::for_kind(kind).requires_api_key); + + if local { + // A local runtime has no credential to supply, so a blank key must + // still leave it dialable rather than skipping it forever. + assert!( + matches!(resolution, Resolution::Ready(_)), + "{} is a local runtime and should resolve as ready without a key", + entry.slug + ); + } else { + assert!( + matches!(resolution, Resolution::Skipped(_)), + "{} should resolve cleanly and skip on a blank key", + entry.slug + ); + } } } + +/// LM Studio must be reachable through the matrix as a **preset**, not just as +/// a bare base URL. +/// +/// The distinction is behavioural, not cosmetic: the preset routes through the +/// transport's local-runtime path (no `Authorization` header, `/v1` base-URL +/// normalisation, and the request-shape degradations llama.cpp-backed servers +/// need), while `PROVIDER_LMSTUDIO_BASE_URL` alone resolves to +/// [`ProviderKind::Compatible`] and gets none of it. +#[test] +fn the_lmstudio_preset_resolves_as_a_keyless_local_runtime() { + let entry = ProviderEntry { + slug: "LMSTUDIO".to_string(), + preset: Some("lmstudio".to_string()), + model: Some("qwen/qwen3-4b".to_string()), + api_key: Some(String::new()), + ..ProviderEntry::default() + }; + + let Resolution::Ready(resolved) = resolve(&entry, &|_| None) else { + panic!("a local runtime should resolve without any credential"); + }; + assert_eq!(resolved.spec.kind, ProviderKind::LmStudio); + assert_eq!(resolved.spec.base_url, "http://localhost:1234/v1"); + assert!(!resolved.spec.requires_api_key); + + // Without a model the preset is incomplete, because LM Studio has no + // default id to fall back on. + let no_model = ProviderEntry { + model: None, + ..entry + }; + assert!(matches!( + resolve(&no_model, &|_| None), + Resolution::Invalid(_) + )); +} From 97169e7fa9bcc1ab0805ff05f0d1e8c12dd9f7ae Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 8 Aug 2026 14:33:41 +0300 Subject: [PATCH 2/4] fix(harness): recover tool arguments buried in an envelope Co-authored-by: Medulla --- src/harness/agent_loop/test.rs | 234 ++++++++++++++++++++++++++++++++ src/harness/agent_loop/tools.rs | 71 ++++++++++ 2 files changed, 305 insertions(+) diff --git a/src/harness/agent_loop/test.rs b/src/harness/agent_loop/test.rs index 9663f56..8011331 100644 --- a/src/harness/agent_loop/test.rs +++ b/src/harness/agent_loop/test.rs @@ -3683,3 +3683,237 @@ async fn an_endless_continue_is_bounded_by_max_model_calls() { "expected the model-call cap to stop it, got: {err}" ); } + +// --------------------------------------------------------------------------- +// Schema-echo argument recovery +// +// Small local models sometimes fill the tool's own JSON Schema in place and +// send the whole envelope as the arguments. These pin the conservative +// unwrap in `normalize_tool_arguments`, in both directions. +// --------------------------------------------------------------------------- + +/// A tool whose arguments genuinely include a field named `properties`, so the +/// echo-unwrap must leave its calls alone. +struct SchemaShapedTool { + calls: Arc>, +} + +#[async_trait] +impl Tool<()> for SchemaShapedTool { + fn name(&self) -> &str { + "schema_shaped" + } + fn description(&self) -> &str { + "takes a literal `properties` argument" + } + fn schema(&self) -> ToolSchema { + ToolSchema::new( + "schema_shaped", + "takes a literal `properties` argument", + json!({ + "type": "object", + "required": ["properties"], + "properties": { + "properties": { "type": "object" } + } + }), + ) + } + async fn call(&self, _state: &(), call: ToolCall) -> Result { + *self.calls.lock().unwrap() += 1; + Ok(ToolResult::text( + call.id, + self.name(), + serde_json::to_string(&call.arguments).unwrap_or_default(), + )) + } +} + +/// A tool that records the arguments it was actually invoked with, so a test +/// can assert what normalization produced rather than only that it ran. +struct ArgumentRecordingTool { + seen: Arc>>, +} + +#[async_trait] +impl Tool<()> for ArgumentRecordingTool { + fn name(&self) -> &str { + "strict_lookup" + } + fn description(&self) -> &str { + "strict lookup" + } + fn schema(&self) -> ToolSchema { + StrictLookupTool { + calls: Arc::new(Mutex::new(0)), + } + .schema() + } + async fn call(&self, _state: &(), call: ToolCall) -> Result { + self.seen.lock().unwrap().push(call.arguments.clone()); + Ok(ToolResult::text(call.id, self.name(), "strict-output")) + } +} + +#[tokio::test] +async fn echoed_schema_arguments_are_unwrapped_before_validation() { + let mut harness: AgentHarness<()> = AgentHarness::new(); + harness.register_model( + "mock", + Arc::new(MockModel::with_responses(vec![ + tool_call_response( + "call-1", + "strict_lookup", + // The exact shape `llama3.2:3b` emits: the declaration filled + // in place, rather than the arguments object. + json!({ + "type": "object", + "required": ["query"], + "properties": { "query": "rust" } + }), + ), + text_response("done", 1, 1), + ])), + ); + let seen = Arc::new(Mutex::new(Vec::new())); + harness.register_tool(Arc::new(ArgumentRecordingTool { + seen: Arc::clone(&seen), + })); + harness.with_policy(RunPolicy { + invalid_args: InvalidArgsPolicy::NormalizeThenReturnToolError, + ..RunPolicy::default() + }); + + let run = harness + .invoke_default(&(), vec![Message::user("lookup")]) + .await + .expect("an echoed schema should be unwrapped, not rejected"); + + assert_eq!(run.final_response.unwrap().text(), "done"); + // The tool ran once, with the unwrapped arguments — not the envelope. + assert_eq!( + *seen.lock().unwrap(), + vec![json!({ "query": "rust" })], + "the envelope should have been unwrapped to the inner arguments" + ); +} + +/// The other envelope shapes captured from `llama3.2:3b`: the arguments nested +/// under `arguments` alongside a schema echo, and under a bare `param` key. +#[tokio::test] +async fn wrapped_arguments_are_unwrapped_from_every_known_envelope_key() { + for envelope in [ + json!({ + "properties": { "query": { "type": "string" } }, + "required": ["query"], + "arguments": { "query": "rust" } + }), + json!({ "param": { "query": "rust" } }), + json!({ "params": { "query": "rust" } }), + json!({ "args": { "query": "rust" } }), + json!({ "parameters": { "query": "rust" } }), + json!({ "input": { "query": "rust" } }), + ] { + let mut harness: AgentHarness<()> = AgentHarness::new(); + harness.register_model( + "mock", + Arc::new(MockModel::with_responses(vec![ + tool_call_response("call-1", "strict_lookup", envelope.clone()), + text_response("done", 1, 1), + ])), + ); + let seen = Arc::new(Mutex::new(Vec::new())); + harness.register_tool(Arc::new(ArgumentRecordingTool { + seen: Arc::clone(&seen), + })); + harness.with_policy(RunPolicy { + invalid_args: InvalidArgsPolicy::NormalizeThenReturnToolError, + ..RunPolicy::default() + }); + + harness + .invoke_default(&(), vec![Message::user("lookup")]) + .await + .unwrap_or_else(|e| panic!("{envelope} should be unwrapped: {e}")); + + assert_eq!( + *seen.lock().unwrap(), + vec![json!({ "query": "rust" })], + "{envelope} should have been unwrapped to the inner arguments" + ); + } +} + +#[tokio::test] +async fn echo_unwrap_leaves_a_tool_that_really_takes_properties_alone() { + let mut harness: AgentHarness<()> = AgentHarness::new(); + harness.register_model( + "mock", + Arc::new(MockModel::with_responses(vec![ + tool_call_response( + "call-1", + "schema_shaped", + json!({ "properties": { "query": "rust" } }), + ), + text_response("done", 1, 1), + ])), + ); + let calls = Arc::new(Mutex::new(0)); + harness.register_tool(Arc::new(SchemaShapedTool { + calls: Arc::clone(&calls), + })); + harness.with_policy(RunPolicy { + invalid_args: InvalidArgsPolicy::NormalizeThenReturnToolError, + ..RunPolicy::default() + }); + + let run = harness + .invoke_default(&(), vec![Message::user("run")]) + .await + .expect("a valid call must not be rewritten"); + + assert_eq!(run.final_response.unwrap().text(), "done"); + assert_eq!(*calls.lock().unwrap(), 1); +} + +#[tokio::test] +async fn echo_unwrap_is_skipped_when_the_inner_value_is_still_invalid() { + let mut harness: AgentHarness<()> = AgentHarness::new(); + harness.register_model( + "mock", + Arc::new(MockModel::with_responses(vec![ + tool_call_response( + "call-1", + "strict_lookup", + // `query` is an integer, so unwrapping would not rescue it. + // The original envelope must survive so the model sees a + // precise validation error rather than a rewritten one. + json!({ + "type": "object", + "properties": { "query": 7 } + }), + ), + text_response("done", 1, 1), + ])), + ); + let calls = Arc::new(Mutex::new(0)); + harness.register_tool(Arc::new(StrictLookupTool { + calls: Arc::clone(&calls), + })); + harness.with_policy(RunPolicy { + invalid_args: InvalidArgsPolicy::NormalizeThenReturnToolError, + ..RunPolicy::default() + }); + + let run = harness + .invoke_default(&(), vec![Message::user("lookup")]) + .await + .expect("the loop recovers by handing the validation error back"); + + assert_eq!(run.final_response.unwrap().text(), "done"); + assert_eq!( + *calls.lock().unwrap(), + 0, + "the tool must not run with arguments that never validated" + ); +} diff --git a/src/harness/agent_loop/tools.rs b/src/harness/agent_loop/tools.rs index 864875c..fb610d2 100644 --- a/src/harness/agent_loop/tools.rs +++ b/src/harness/agent_loop/tools.rs @@ -549,6 +549,7 @@ fn normalize_tool_arguments(call: &mut ToolCall, schema: &ToolSchema) { // recover. If its contents violate the schema, preserve them so the model // sees the real validation error instead of executing with an empty object. if call.arguments.is_object() { + unwrap_wrapped_arguments(call, schema); return; } @@ -561,6 +562,76 @@ fn normalize_tool_arguments(call: &mut ToolCall, schema: &ToolSchema) { } } +/// Keys under which a model commonly buries the real arguments object. +/// +/// `properties` is the JSON-Schema echo; the rest are the wrapper names small +/// models invent when they confuse the *call* envelope with its payload. All of +/// them were observed on local runtimes — see [`unwrap_wrapped_arguments`]. +const ARGUMENT_WRAPPER_KEYS: [&str; 7] = [ + "properties", + "arguments", + "args", + "parameters", + "params", + "param", + "input", +]; + +/// Recovers arguments a model buried one level deep inside an envelope. +/// +/// Small local models (observed on `llama3.2:3b` via Ollama) routinely send +/// something other than a bare arguments object. All three of these are real +/// captures for a tool declaring one required `city` string: +/// +/// ```text +/// {"type":"object","required":["city"],"properties":{"city":"Paris"}} +/// {"properties":{...},"required":[...],"arguments":{"city":"Paris"}} +/// {"param":{"city":"Paris"}} +/// ``` +/// +/// In each case the intended `{"city":"Paris"}` is present, one level down. +/// Without this the call fails validation, costs a repair round trip, and on +/// the default [`InvalidArgsPolicy::Fail`] aborts the run outright. +/// +/// The rewrite is deliberately conservative and cannot corrupt a legitimate +/// call. For each candidate key it applies only when the outer object is +/// already schema-invalid, when the tool does not itself declare an argument of +/// that name (so the key is not meaningfully the model's own data), and when +/// the unwrapped value *does* validate. If no candidate satisfies all three the +/// original arguments are left untouched, so the model still sees a precise +/// validation error rather than a rewritten one. +/// +/// [`InvalidArgsPolicy::Fail`]: crate::harness::runtime::InvalidArgsPolicy::Fail +fn unwrap_wrapped_arguments(call: &mut ToolCall, schema: &ToolSchema) { + let declared = schema + .parameters + .get("properties") + .and_then(Value::as_object); + + for key in ARGUMENT_WRAPPER_KEYS { + // A tool that genuinely takes an argument of this name must never have + // it unwrapped — for such a tool the key is data, not an envelope. + if declared.is_some_and(|declared| declared.contains_key(key)) { + continue; + } + let Some(inner) = call + .arguments + .get(key) + .filter(|inner| inner.is_object()) + .cloned() + else { + continue; + }; + + let mut candidate = call.clone(); + candidate.arguments = inner; + if schema.validate_call(&candidate).is_ok() { + call.arguments = candidate.arguments; + return; + } + } +} + fn strip_markdown_code_fence(raw: &str) -> &str { let trimmed = raw.trim(); let Some(after_open) = trimmed.strip_prefix("```") else { From 7241d638258e304bce486f729466b88967ea4a85 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 8 Aug 2026 14:33:42 +0300 Subject: [PATCH 3/4] fix(providers): recover tool calls local models emit as text Co-authored-by: Medulla --- src/harness/providers/openai/mod.rs | 2 +- src/harness/providers/openai/relaxed_json.rs | 109 ++++++++++++++++++- src/harness/tool/prompt.rs | 99 ++++++++++++++++- src/harness/tool/prompt_test.rs | 104 ++++++++++++++++++ 4 files changed, 306 insertions(+), 8 deletions(-) diff --git a/src/harness/providers/openai/mod.rs b/src/harness/providers/openai/mod.rs index b46a2a8..4a7bc8a 100644 --- a/src/harness/providers/openai/mod.rs +++ b/src/harness/providers/openai/mod.rs @@ -85,7 +85,7 @@ const DEFAULT_REQUEST_TIMEOUT_SECS: u64 = 600; mod convert; mod reasoning_tags; -mod relaxed_json; +pub(crate) mod relaxed_json; mod responses; mod sse; mod transport; diff --git a/src/harness/providers/openai/relaxed_json.rs b/src/harness/providers/openai/relaxed_json.rs index 13672e8..d234b61 100644 --- a/src/harness/providers/openai/relaxed_json.rs +++ b/src/harness/providers/openai/relaxed_json.rs @@ -64,7 +64,7 @@ const LEAKED_QUOTE_TOKENS: &[&str] = &["<|\"|>", "<|\">"]; /// /// See the module docs for the repair strategy and the safety invariant (only /// invoked after strict parsing has already failed). -pub(super) fn recover_relaxed_object(raw: &str) -> Option { +pub(crate) fn recover_relaxed_object(raw: &str) -> Option { let normalized = normalize_leaked_quote_tokens(raw); let mut layer = normalized.trim().to_string(); for _ in 0..=MAX_BRACE_PEEL { @@ -184,6 +184,51 @@ enum Container { /// identifiers in array or value position are left alone (so `["discord"]`, /// `true`, numbers, and already-quoted keys pass through unchanged). Returns the /// input verbatim when there is nothing to quote. +/// Reads a quote-delimited object key whose delimiters may be single quotes or +/// mismatched, returning the key text and the bytes consumed (including both +/// delimiters). +/// +/// `rest` begins at the opening quote. Models that lose track of their own +/// string delimiters produce `'city'`, `"city'`, and `'city"` interchangeably — +/// all three mean the same key, and strict JSON accepts none of them. +/// +/// Returns `None` for a well-formed `"key"` so the caller keeps using the +/// normal in-string path, and `None` for anything that does not look like a +/// key: the token must be terminated by `'` or `"` followed (after optional +/// whitespace) by a `:`, and must not span a line break or contain structural +/// JSON characters. That keeps a legitimate double-quoted key containing an +/// apostrophe (`{"it's fine": 1}`) from being truncated at the apostrophe, +/// because there the next character after `'` is not a colon. +fn take_quoted_key(rest: &str) -> Option<(String, usize)> { + let mut chars = rest.char_indices(); + let (_, open) = chars.next()?; + debug_assert!(open == '"' || open == '\''); + + let mut key = String::new(); + for (idx, ch) in chars { + match ch { + '"' | '\'' => { + let after = &rest[idx + ch.len_utf8()..]; + if after.trim_start().starts_with(':') { + // A perfectly well-formed key needs no rewriting; let the + // ordinary scanner handle it so behaviour is unchanged. + if open == '"' && ch == '"' { + return None; + } + return Some((key, idx + ch.len_utf8())); + } + // Not the end of a key — record it and keep looking. + key.push(ch); + } + // A key never spans a newline or contains structure; bail out and + // let the ordinary scanner deal with whatever this really is. + '\n' | '\r' | '{' | '}' | '[' | ']' | ':' => return None, + _ => key.push(ch), + } + } + None +} + fn quote_bare_keys(s: &str) -> String { let mut out = String::with_capacity(s.len() + 8); let mut stack: Vec = Vec::new(); @@ -206,6 +251,29 @@ fn quote_bare_keys(s: &str) -> String { } match ch { + // A quote in key position may open a *mismatched* key delimiter + // (`"city'`) or a single-quoted one (`'city'`), neither of which the + // in-string scanner below can terminate correctly. Try that first; + // a well-formed `"key"` falls through to the normal path. + '"' | '\'' if expect_key && matches!(stack.last(), Some(Container::Object)) => { + match take_quoted_key(&s[idx..]) { + Some((key, consumed)) => { + out.push('"'); + out.push_str(&key.replace('\\', r"\\").replace('"', "\\\"")); + out.push('"'); + // Advance the iterator past the bytes just consumed. + while chars.peek().is_some_and(|&(next, _)| next < idx + consumed) { + chars.next(); + } + expect_key = false; + } + None => { + in_string = true; + expect_key = false; + out.push(ch); + } + } + } '"' => { in_string = true; expect_key = false; @@ -279,6 +347,45 @@ mod tests { use super::*; use serde_json::json; + #[test] + fn repairs_single_quoted_and_mismatched_keys() { + // Captured from `llama3.2:3b` via Ollama: the model loses track of its + // own string delimiters mid-object. + assert_eq!( + recover_relaxed_object(r#"{"name":"get_weather","parameters':{'city':"Paris"}}"#), + Some(json!({ "name": "get_weather", "parameters": { "city": "Paris" } })) + ); + // Single-quoted keys are repaired the same way, as long as the values + // themselves are well-formed. + assert_eq!( + recover_relaxed_object(r#"{'city':"Paris"}"#), + Some(json!({ "city": "Paris" })) + ); + } + + /// Single-quoted *values* are deliberately **not** repaired. + /// + /// A key is a short identifier, so reading `'` as a delimiter there is + /// safe. A value is free text where an apostrophe is ordinary English + /// (`"it's sunny"`), and treating those as delimiters would corrupt real + /// arguments. Such a blob stays unrecovered, the call is marked invalid, + /// and the agent loop hands the model a precise error to retry against — + /// the same path every other unrepairable blob takes. + #[test] + fn single_quoted_values_are_left_unrepaired() { + assert_eq!(recover_relaxed_object(r#"{'city':'Paris'}"#), None); + } + + #[test] + fn an_apostrophe_inside_a_well_formed_key_is_not_a_delimiter() { + // `'` here is followed by ` fine"`, not a colon, so the key survives + // whole rather than being truncated at the apostrophe. + assert_eq!( + recover_relaxed_object(r#"{"it's fine":1,bare:2}"#), + Some(json!({ "it's fine": 1, "bare": 2 })) + ); + } + #[test] fn quotes_unquoted_keys() { assert_eq!( diff --git a/src/harness/tool/prompt.rs b/src/harness/tool/prompt.rs index 3a5142e..6c9f62b 100644 --- a/src/harness/tool/prompt.rs +++ b/src/harness/tool/prompt.rs @@ -480,8 +480,22 @@ pub fn should_recover(native: bool, has_tools: bool, structured_calls: usize) -> /// using [`with_prompt_tool_instructions`]. pub fn apply_prompt_tool_calls(mut response: ModelResponse) -> ModelResponse { let text = response.text(); - let (cleaned, calls) = parse_prompt_tool_calls_from_text(&text); + let (cleaned, mut calls) = parse_prompt_tool_calls_from_text(&text); if calls.is_empty() { + // No delimited block. A small local model may still have emitted the + // call as a bare object with no markup at all — see + // `parse_bare_tool_call`. That path consumes the whole content, so the + // cleaned prose is empty by construction. + if let Some(call) = parse_bare_tool_call(&text) { + calls.push(call); + response.message.tool_calls.extend(calls); + // The object was the whole visible text, so nothing survives as + // prose — but a reasoning model's `Thinking` block must, hence + // `replace_text_blocks` with empty text rather than clearing the + // content outright. + response.message.content = replace_text_blocks(response.message.content, String::new()); + return response; + } return response; } response.message.tool_calls.extend(calls); @@ -516,13 +530,41 @@ fn replace_text_blocks(content: Vec, cleaned: String) -> Vec Option { - let value: Value = serde_json::from_str(inner).ok()?; - let name = value.get("name")?.as_str()?.to_string(); - let arguments = value - .get("arguments") - .cloned() + let value = parse_relaxed_object(inner)?; + tool_call_from_object(&value, index) +} + +/// Parses a JSON object, repairing the relaxed spellings small local models +/// emit (unquoted keys, redundant braces, leaked quote tokens) when strict +/// parsing fails. +fn parse_relaxed_object(raw: &str) -> Option { + match serde_json::from_str::(raw) { + Ok(value) if value.is_object() => Some(value), + // A non-object parsed strictly is not a tool call; do not try to + // "repair" it into one. + Ok(_) => None, + Err(_) => crate::harness::providers::openai::relaxed_json::recover_relaxed_object(raw), + } +} + +/// Builds a [`ToolCall`] from an already-parsed call object, or `None` when the +/// object does not name a tool. +fn tool_call_from_object(value: &Value, index: usize) -> Option { + let name = value.get("name")?.as_str()?.trim().to_string(); + if name.is_empty() { + return None; + } + let arguments = CALL_ARGUMENT_KEYS + .iter() + .find_map(|key| value.get(*key).cloned()) .unwrap_or_else(|| Value::Object(Map::new())); Some(ToolCall { id: format!("call_{index}"), @@ -531,3 +573,48 @@ fn parse_one(inner: &str, index: usize) -> Option { invalid: None, }) } + +/// Recovers a tool call a model emitted as a **bare object**, with no +/// `` markup of any kind. +/// +/// Observed on `llama3.2:3b` via Ollama under `tool_choice: "required"`: rather +/// than populating the wire's `tool_calls` array, roughly one response in a +/// dozen puts the call in `content` as +/// +/// ```text +/// {"name":"get_weather","parameters':{'city':"Paris"}} +/// ``` +/// +/// — note the mismatched quotes, which strict JSON also rejects. Without +/// recovery the agent loop sees an assistant message with no tool calls, treats +/// it as the final answer, and silently returns JSON-looking prose to the user +/// instead of running the tool. +/// +/// # Why this cannot swallow a genuine text answer +/// +/// The recovery requires the **entire** message content (trimmed, and with a +/// surrounding markdown fence removed) to parse as a single JSON object +/// carrying a string `name`. Prose that merely mentions or quotes JSON has text +/// outside the object and is left untouched, as is any object that does not +/// name a tool. The caller only reaches this path when the request declared +/// tools and the response carried no structured tool calls. +fn parse_bare_tool_call(text: &str) -> Option { + let candidate = strip_code_fence(text.trim()); + if !(candidate.starts_with('{') && candidate.ends_with('}')) { + return None; + } + let value = parse_relaxed_object(candidate)?; + tool_call_from_object(&value, 1) +} + +/// Strips one surrounding markdown code fence, with or without a language tag. +fn strip_code_fence(raw: &str) -> &str { + let Some(after_open) = raw.strip_prefix("```") else { + return raw; + }; + let body = match after_open.find('\n') { + Some(newline) => &after_open[newline + 1..], + None => return raw, + }; + body.trim_end().strip_suffix("```").map_or(raw, str::trim) +} diff --git a/src/harness/tool/prompt_test.rs b/src/harness/tool/prompt_test.rs index ef5d5c2..fcbca6d 100644 --- a/src/harness/tool/prompt_test.rs +++ b/src/harness/tool/prompt_test.rs @@ -510,3 +510,107 @@ fn apply_prompt_tool_calls_preserves_a_leading_thinking_block() { ContentBlock::Text("reply".to_string()) ); } +// --------------------------------------------------------------------------- +// Bare (undelimited) tool calls +// +// Captured from `llama3.2:3b` via Ollama with `tool_choice: "required"`: the +// model puts the call in `content` instead of the wire's `tool_calls` array, +// with no `` markup and frequently with malformed JSON. +// --------------------------------------------------------------------------- + +#[test] +fn apply_prompt_tool_calls_recovers_a_bare_object_with_relaxed_json() { + // The exact capture: `parameters'` and `{'city'` use mismatched quotes, so + // strict JSON rejects it outright. + let resp = crate::harness::model::ModelResponse::assistant( + r#"{"name":"get_weather","parameters':{'city':"Paris"}}"#, + ); + let out = apply_prompt_tool_calls(resp); + + assert_eq!(out.message.tool_calls.len(), 1); + assert_eq!(out.message.tool_calls[0].name, "get_weather"); + assert_eq!( + out.message.tool_calls[0].arguments, + serde_json::json!({ "city": "Paris" }) + ); + // The raw markup must not also survive as prose, or the user sees the JSON. + assert!( + out.text().is_empty(), + "the consumed object should not remain as text: {}", + out.text() + ); +} + +#[test] +fn apply_prompt_tool_calls_recovers_a_bare_object_inside_a_code_fence() { + let resp = crate::harness::model::ModelResponse::assistant( + "```json\n{\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}\n```", + ); + let out = apply_prompt_tool_calls(resp); + + assert_eq!(out.message.tool_calls.len(), 1); + assert_eq!(out.message.tool_calls[0].name, "get_weather"); +} + +#[test] +fn a_tool_call_object_may_name_its_arguments_parameters() { + let resp = crate::harness::model::ModelResponse::assistant( + r#"{"name":"get_weather","parameters":{"city":"Paris"}}"#, + ); + let out = apply_prompt_tool_calls(resp); + + assert_eq!(out.message.tool_calls.len(), 1); + assert_eq!( + out.message.tool_calls[0].arguments, + serde_json::json!({ "city": "Paris" }) + ); +} + +#[test] +fn bare_object_recovery_never_swallows_a_genuine_text_answer() { + // Prose, prose that merely quotes JSON, a JSON object that names no tool, + // and a bare JSON scalar must all pass through untouched. + for text in [ + "The weather in Paris is mild today.", + r#"You could send {"name":"get_weather"} to that endpoint."#, + r#"{"city":"Paris","temperature":17}"#, + r#"{"name":42}"#, + r#""just a string""#, + "[1, 2, 3]", + ] { + let out = apply_prompt_tool_calls(crate::harness::model::ModelResponse::assistant(text)); + assert!( + out.message.tool_calls.is_empty(), + "{text:?} must not be recovered as a tool call" + ); + assert_eq!(out.text(), text, "{text:?} must survive as text"); + } +} + +#[test] +fn bare_tool_call_recovery_preserves_a_thinking_block() { + // A local *reasoning* model emits its chain of thought and then the bare + // call object as the whole visible text. Consuming the object must not take + // the reasoning with it. + let mut response = ModelResponse::assistant(r#"{"name":"search","arguments":{"q":"x"}}"#); + response.message.content.insert( + 0, + ContentBlock::Thinking { + text: "chain of thought".to_string(), + signature: None, + }, + ); + + let out = apply_prompt_tool_calls(response); + + assert_eq!(out.message.tool_calls.len(), 1); + assert_eq!(out.message.tool_calls[0].name, "search"); + assert_eq!( + out.message.content, + vec![ContentBlock::Thinking { + text: "chain of thought".to_string(), + signature: None, + }], + "the reasoning must survive while the consumed object does not" + ); +} From 3cc45b6f1b2394aa6d99bf23b33e6e54499f79e6 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 8 Aug 2026 14:33:46 +0300 Subject: [PATCH 4/4] test(harness): live Ollama + LM Studio model and embedding coverage Co-authored-by: Medulla --- docs/modules/harness/README.md | 1 + docs/modules/harness/local-models.md | 198 +++++++ tests/live_local_embeddings.rs | 593 +++++++++++++++++++ tests/live_local_models.rs | 819 +++++++++++++++++++++++++++ 4 files changed, 1611 insertions(+) create mode 100644 docs/modules/harness/local-models.md create mode 100644 tests/live_local_embeddings.rs create mode 100644 tests/live_local_models.rs diff --git a/docs/modules/harness/README.md b/docs/modules/harness/README.md index 942e949..c24f834 100644 --- a/docs/modules/harness/README.md +++ b/docs/modules/harness/README.md @@ -235,6 +235,7 @@ Feature details: - [Context feature](context.md) - [Model and provider feature](model.md) +- [Local models and embeddings (Ollama, LM Studio)](local-models.md) - [Embeddings and retrieval feature](embeddings.md) - [State graph runtime feature](state-graph.md) - [Prompt feature](prompt.md) diff --git a/docs/modules/harness/local-models.md b/docs/modules/harness/local-models.md new file mode 100644 index 0000000..9bf26e3 --- /dev/null +++ b/docs/modules/harness/local-models.md @@ -0,0 +1,198 @@ +# Local Models and Embeddings + +Running against a local runtime — Ollama or LM Studio — is not the same as +running against a hosted provider with a different base URL. The wire format is +identical, but small quantised models and llama.cpp-backed servers fail in ways +the hosted APIs never do, and a host that ignores those differences gets a loop +that works in development and stalls in production. + +This page documents what actually differs, what the crate already handles, and +what a caller still has to configure. Everything here is asserted by +[`tests/live_local_models.rs`](../../../tests/live_local_models.rs) and +[`tests/live_local_embeddings.rs`](../../../tests/live_local_embeddings.rs) +against real servers. + +## Presets + +| Preset | Default base URL | Default model | Credential | +| ------ | ---------------- | ------------- | ---------- | +| `ProviderKind::Ollama` | `http://localhost:11434/v1` | `llama3.2` | none | +| `ProviderKind::LmStudio` | `http://localhost:1234/v1` | **none — you must set one** | none | + +Both resolve through `OpenAiModel::from_spec`, which routes local kinds onto the +local-runtime path: `AuthStyle::None` (no `Authorization` header at all, which +some servers reject), base-URL normalisation to the `/v1` root, and the +request-shape degradations described below. + +Reaching LM Studio through a bare `PROVIDER_LMSTUDIO_BASE_URL` instead of the +preset resolves to `ProviderKind::Compatible` and gets **none** of that. Use the +preset. + +LM Studio has no default model on purpose: the served id is whatever GGUF the +operator loaded, so any guess would 404 on most installs. Discover it at runtime: + +```rust +let spec = ProviderSpec::for_kind(ProviderKind::LmStudio).with_model("probe"); +let ids = OpenAiModel::from_spec(spec, "local")?.list_models().await?; +``` + +## What breaks, and where it is handled + +### Request shapes local servers reject + +A named `tool_choice` object and `response_format: {"type": "json_object"}` both +draw an HTTP 400 from llama.cpp-backed servers. The transport degrades them to +shapes those servers accept — `tool_choice: "required"` with the `tools` array +filtered to the named tool, and a permissive `json_schema` — either eagerly for +local presets or as a single retry when a 400 body implicates the shape. + +### Tool arguments that are not the arguments + +Small models frequently send something other than a bare arguments object. All +of these are real captures from `llama3.2:3b` for a tool declaring one required +`city` string: + +```text +{"type":"object","required":["city"],"properties":{"city":"Paris"}} # schema echo +{"properties":{…},"required":[…],"arguments":{"city":"Paris"}} # nested under `arguments` +{"param":{"city":"Paris"}} # invented wrapper +``` + +`normalize_tool_arguments` unwraps a single envelope level for a known set of +wrapper keys, but only when the outer object is already schema-invalid, the tool +does not itself declare an argument of that name, and the unwrapped value +validates. Failing any of those, the original arguments survive so the model +sees a precise error rather than a rewritten one. + +This only runs under a recovering `InvalidArgsPolicy` — see below. + +### Tool calls emitted as text + +Roughly one response in a dozen, `llama3.2:3b` under `tool_choice: "required"` +puts the call in `content` instead of the wire's `tool_calls` array, with no +`` markup and often malformed JSON: + +```text +{"name":"get_weather","parameters':{'city':"Paris"}} +``` + +Left alone this is catastrophic rather than merely lossy: the loop sees an +assistant message with no tool calls, treats it as the final answer, and returns +JSON-looking prose to the user while the tool never runs. +`apply_prompt_tool_calls` recovers it, requiring the **entire** message content +to parse as one object naming a tool so prose that merely quotes JSON is never +swallowed. Mismatched and single-quoted *keys* are repaired; single-quoted +*values* deliberately are not, because an apostrophe in a value is ordinary +English. + +### Invalid arguments abort the run by default + +`RunPolicy::invalid_args` defaults to `InvalidArgsPolicy::Fail`: the first +schema-invalid tool call kills the whole run. That is defensible for a frontier +model, where such a call is nearly always a genuine bug. For a 3B model it makes +the loop unusable — and it disables the argument recovery above, which only runs +under the recovering policy. + +**A host driving a local model should opt in:** + +```rust +harness.with_policy(RunPolicy { + invalid_args: InvalidArgsPolicy::NormalizeThenReturnToolError, + ..RunPolicy::default() +}); +``` + +Recovery still consumes a tool-call budget slot, so `RunLimits::max_tool_calls` +bounds any repair loop. + +### Reasoning models and the token budget + +`qwen3` (both runtimes) spends tokens on a hidden reasoning channel before +emitting a single visible character, and that channel draws from the same +`max_tokens` budget. Asking `qwen3-4b` for one word with `max_tokens: 32` +returns `finish_reason: "length"`, 30 reasoning tokens, and **empty** visible +content. + +Budget for the reasoning channel — the live tests use 1024 tokens for prompts +whose answers are a few words. `RunPolicy::empty_response_retries` exists for the +residual stochastic case where a model burns the whole budget anyway. + +Inline `` blocks are extracted into `ContentBlock::Thinking` rather than +leaking into the visible answer; see `ReasoningTagExtraction`. + +### Tool choice is a request, not a guarantee + +`ToolChoice::Required` is honoured by Ollama roughly eleven times in twelve for +`llama3.2:3b`; the remainder emit the call as text (recovered as above) or +answer directly. Any host that depends on a tool actually running must check, +not assume. + +## Embeddings + +The two runtimes are reached by **different adapters**, because Ollama does not +serve embeddings on its OpenAI-compatible surface: + +| Runtime | Adapter | Endpoint | Base URL | +| ------- | ------- | -------- | -------- | +| Ollama | `OllamaEmbeddingModel` | `POST /api/embed` | server **root**, e.g. `http://localhost:11434` | +| LM Studio | `OpenAiEmbeddingModel` | `POST /v1/embeddings` | `http://localhost:1234/v1` | + +`OllamaEmbeddingModel` rejects a base URL carrying a `/v1` or `/api` suffix at +construction, because the chat side of the *same server* is configured with +`/v1` and copying it there yields a 404 on every call. + +### Discover the width; do not declare it + +Vector width is a property of the installed GGUF. `nomic-embed-text` is 768 wide +while `OllamaEmbeddingModel`'s own default (`bge-m3`) is 1024, and a declared +width that disagrees with the model fails every call on dimension validation. + +- Ollama: `OllamaEmbeddingModel::embed_discovering_dimensions(...)`. Passing `0` + to `try_new` does **not** mean "discover" — it means "use the default". +- OpenAI-compatible: construct `with_dimensions(0)` to disable validation, probe, + then rebuild with the observed width. Also set `with_send_dimensions(false)`: + `dimensions` is an OpenAI request parameter for Matryoshka truncation that + llama.cpp-backed servers reject or ignore. + +Width matters beyond the immediate call: `InMemoryVectorStore` fixes its width on +first insert, and `EmbeddingModel::signature()` embeds the width to partition +persisted vectors between embedding spaces. + +### Blank inputs diverge between adapters + +Both adapters are position-safe — neither silently drops a blank and shifts every +later vector onto the wrong id — but they achieve it differently: + +- `OllamaEmbeddingModel` returns an empty vector per blank input, preserving + positions, without dialling the server. +- `OpenAiEmbeddingModel` rejects the whole batch with a validation error naming + the offending index. + +They are therefore **not** interchangeable: code that indexes a corpus containing +blanks works against Ollama and fails against any OpenAI-compatible endpoint, +LM Studio included. Filter blanks in the caller rather than relying on either. + +## Running the tests + +```bash +# Ollama +ollama serve & +ollama pull llama3.2:3b && ollama pull nomic-embed-text + +# LM Studio +lms server start +lms load qwen/qwen3-4b && lms load text-embedding-nomic-embed-text-v1.5 + +LOCAL_MODEL_TESTS=1 cargo test --test live_local_models --test live_local_embeddings -- --nocapture +``` + +Both files skip entirely without `LOCAL_MODEL_TESTS=1`, and skip any individual +runtime that is not listening. With the variable set and **nothing** reachable +they fail rather than pass, because every assertion lives inside a loop over the +reachable runtimes — an empty list would otherwise be a green run that tested +nothing. + +Per-runtime overrides: `LOCAL_OLLAMA_BASE_URL`, `LOCAL_OLLAMA_MODEL`, +`LOCAL_OLLAMA_EMBED_URL`, `LOCAL_OLLAMA_EMBED_MODEL`, `LOCAL_LMSTUDIO_BASE_URL`, +`LOCAL_LMSTUDIO_MODEL`, `LOCAL_LMSTUDIO_EMBED_MODEL`. Models are otherwise +discovered from each server's own `/v1/models`. diff --git a/tests/live_local_embeddings.rs b/tests/live_local_embeddings.rs new file mode 100644 index 0000000..c3f40be --- /dev/null +++ b/tests/live_local_embeddings.rs @@ -0,0 +1,593 @@ +//! LIVE local-embedding coverage: Ollama's native `/api/embed` and LM Studio's +//! OpenAI-compatible `/v1/embeddings`, end to end through [`Retriever`]. +//! +//! `e2e_embeddings.rs` already proves the retrieval *plumbing* — but it does so +//! against [`MockEmbeddingModel`], which hashes text to a stable vector. That +//! makes the ranking assertion tautological: querying with a document's exact +//! text scores `1.0` by construction, whatever the embedding space is like. It +//! cannot catch a real embedding adapter that returns vectors in the wrong +//! order, silently truncates a batch, reports a dimensionality that disagrees +//! with the vectors it produces, or returns embeddings so degenerate that +//! retrieval is no better than chance. +//! +//! So this file drives real local embedding servers and asserts the properties +//! that actually matter for retrieval: +//! +//! - **dimensional honesty** — `dimensions()` agrees with every vector emitted, +//! and stays stable across calls, because the vector store partitions on it, +//! - **positional integrity** — the *n*th vector belongs to the *n*th input, +//! asserted by embedding a batch and comparing against one-at-a-time calls, +//! - **semantic separation** — a paraphrase scores higher than an unrelated +//! sentence, so the vectors carry meaning rather than noise, and +//! - **retrieval** — a real query against a real index returns the right +//! document first, with no exact-text shortcut. +//! +//! # Configuration +//! +//! | Variable | Default | +//! |---|---| +//! | `LOCAL_MODEL_TESTS` | unset — **required** to dial anything | +//! | `LOCAL_OLLAMA_EMBED_URL` | `http://localhost:11434` (server root, not `/v1`) | +//! | `LOCAL_OLLAMA_EMBED_MODEL` | `nomic-embed-text` | +//! | `LOCAL_LMSTUDIO_BASE_URL` | `http://localhost:1234/v1` | +//! | `LOCAL_LMSTUDIO_EMBED_MODEL` | first embedding id `GET /v1/models` advertises | +//! +//! # Skips gracefully +//! +//! Opt-in via `LOCAL_MODEL_TESTS=1`, and any server that is not listening — or +//! has no embedding model installed — is reported and skipped rather than +//! failed. +//! +//! # Run +//! +//! ```text +//! LOCAL_MODEL_TESTS=1 cargo test --test live_local_embeddings -- --nocapture +//! ``` + +use std::sync::Arc; + +use serde_json::json; +use tinyagents::harness::embeddings::{ + EmbeddingModel, InMemoryVectorStore, OllamaEmbeddingModel, OpenAiEmbeddingModel, + RECOMMENDED_OLLAMA_CONTEXT_TOKENS, Retriever, cosine_similarity, +}; +use tinyagents::harness::providers::openai::OpenAiModel; +use tinyagents::harness::providers::{ProviderKind, ProviderSpec}; + +/// A local embedding backend under test, behind the provider-neutral trait so +/// every assertion below is written once and runs against both. +struct LocalEmbedder { + name: &'static str, + model: Arc, +} + +/// Ollama exposes embeddings on its **native** `/api/embed`, not the +/// OpenAI-compatible surface, so it gets the dedicated adapter and its base URL +/// is the server root (`OllamaEmbeddingModel` rejects a `/v1` or `/api` suffix). +async fn ollama_embedder() -> std::result::Result { + let base_url = env_or("LOCAL_OLLAMA_EMBED_URL", "http://localhost:11434"); + let model_id = env_or("LOCAL_OLLAMA_EMBED_MODEL", "nomic-embed-text"); + + // The width is a property of the installed GGUF, not something a caller can + // know: `nomic-embed-text` is 768 wide while the adapter's own default + // (`bge-m3`) is 1024, and declaring the wrong one makes every call fail + // dimension validation. `embed_discovering_dimensions` is the supported way + // to learn it — passing `0` to `try_new` does *not* mean "discover", it + // means "use the default". This probe therefore doubles as the reachability + // and model-installed check. + let (width, _) = OllamaEmbeddingModel::embed_discovering_dimensions( + &base_url, + &model_id, + reqwest::Client::new(), + &["probe".to_string()], + RECOMMENDED_OLLAMA_CONTEXT_TOKENS, + RECOMMENDED_OLLAMA_CONTEXT_TOKENS, + ) + .await + .map_err(|e| format!("ollama embeddings ({model_id}): {e}"))?; + + let model = OllamaEmbeddingModel::try_new(&base_url, &model_id, width) + .map_err(|e| format!("ollama: invalid embedding configuration: {e}"))?; + + Ok(LocalEmbedder { + name: "ollama", + model: Arc::new(model), + }) +} + +/// LM Studio serves embeddings on the OpenAI-compatible `/v1/embeddings`, so the +/// hosted adapter reaches it with only the base URL changed — and the API key +/// requirement switched off, since a local server neither needs nor checks one. +async fn lmstudio_embedder() -> std::result::Result { + let base_url = env_or("LOCAL_LMSTUDIO_BASE_URL", "http://localhost:1234/v1"); + + let model_id = match std::env::var("LOCAL_LMSTUDIO_EMBED_MODEL") + .ok() + .filter(|v| !v.trim().is_empty()) + { + Some(explicit) => explicit.trim().to_string(), + None => discover_lmstudio_embedding_model(&base_url).await?, + }; + + let configure = |dimensions: usize| { + OpenAiEmbeddingModel::new("local") + .with_base_url(&base_url) + .with_model(&model_id) + .with_required_api_key(false) + // `dimensions` is an OpenAI-specific *request* parameter for + // Matryoshka-style truncation; llama.cpp-backed servers reject or + // ignore it, and the width is whatever the GGUF produces regardless. + .with_send_dimensions(false) + .with_dimensions(dimensions) + }; + + // Discover the width before declaring one. The adapter defaults to + // `text-embedding-3-small`'s 1536 and validates every vector against it, so + // probing with the default would reject a 768-wide local model as a + // mismatch. Zero disables that check, which is exactly what a discovery + // probe needs. + let probe = configure(0) + .embed(&["probe".to_string()]) + .await + .map_err(|e| format!("lmstudio embeddings ({model_id}): {e}"))?; + let width = probe.first().map(Vec::len).unwrap_or(0); + if width == 0 { + return Err(format!("lmstudio: `{model_id}` returned an empty vector")); + } + + Ok(LocalEmbedder { + name: "lmstudio", + model: Arc::new(configure(width)), + }) +} + +/// Asks LM Studio which models it serves and picks an embedding one. +/// +/// There is no default to hard-code: the id is whatever GGUF the operator +/// loaded. +async fn discover_lmstudio_embedding_model(base_url: &str) -> std::result::Result { + let spec = ProviderSpec::for_kind(ProviderKind::LmStudio) + .with_base_url(base_url) + .with_model("probe"); + let client = OpenAiModel::from_spec(spec, "local") + .map_err(|e| format!("lmstudio: invalid configuration: {e}"))?; + + let listed = client + .list_models() + .await + .map_err(|e| format!("lmstudio: not reachable at {base_url} ({e})"))?; + + listed + .iter() + .map(|entry| entry.id.clone()) + .find(|id| { + let id = id.to_ascii_lowercase(); + id.contains("embed") || id.contains("bge") + }) + .ok_or_else(|| { + format!( + "lmstudio: reachable at {base_url} but serves no embedding model \ + (saw {} id(s)); load one or set LOCAL_LMSTUDIO_EMBED_MODEL", + listed.len() + ) + }) +} + +fn env_or(name: &str, default: &str) -> String { + std::env::var(name) + .ok() + .filter(|v| !v.trim().is_empty()) + .unwrap_or_else(|| default.to_string()) +} + +/// Every local embedding backend that is reachable and usable right now. +async fn reachable_embedders() -> Vec { + if std::env::var("LOCAL_MODEL_TESTS") + .ok() + .filter(|v| !v.trim().is_empty() && v != "0") + .is_none() + { + eprintln!( + "skipping live local-embedding tests: set LOCAL_MODEL_TESTS=1 to dial local servers \ + (LOCAL_MODEL_TESTS=1 cargo test --test live_local_embeddings -- --nocapture)" + ); + return Vec::new(); + } + + let mut ready = Vec::new(); + for probe in [ollama_embedder().await, lmstudio_embedder().await] { + match probe { + Ok(embedder) => { + eprintln!( + "local embedder `{}` ready: {} ({} dims)", + embedder.name, + embedder.model.model_id(), + embedder.model.dimensions() + ); + ready.push(embedder); + } + Err(reason) => eprintln!("skipping {reason}"), + } + } + // Opting in explicitly and then reaching nothing must not look like success. + // Every assertion in this file is inside a `for` over this list, so an empty + // list makes the whole suite pass while testing nothing at all — the exact + // failure mode these tests exist to rule out. + assert!( + !ready.is_empty(), + "LOCAL_MODEL_TESTS is set but no local embedding server is reachable. \ + Start Ollama (`ollama serve` + `ollama pull nomic-embed-text`) or LM Studio \ + (`lms server start` + load an embedding model), or unset LOCAL_MODEL_TESTS to skip." + ); + ready +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +/// The reported dimensionality must match the vectors actually produced, and +/// must not drift between calls. +/// +/// This is load-bearing rather than cosmetic: [`InMemoryVectorStore`] fixes its +/// width on the first insert and rejects mismatches, and +/// [`EmbeddingModel::signature`] — which embeds `dimensions()` — is what +/// partitions persisted vectors between embedding spaces. A model whose +/// declared width disagrees with its output silently corrupts both. +#[tokio::test] +async fn local_embedders_report_the_width_they_actually_produce() { + for embedder in reachable_embedders().await { + let declared = embedder.model.dimensions(); + assert!( + declared > 0, + "{}: dimensions() must be resolved after a successful embed", + embedder.name + ); + + let vectors = embedder + .model + .embed(&["alpha".to_string(), "beta".to_string()]) + .await + .unwrap_or_else(|e| panic!("{}: embed failed: {e}", embedder.name)); + + for (index, vector) in vectors.iter().enumerate() { + assert_eq!( + vector.len(), + declared, + "{}: vector {index} is {} wide but the model declares {declared}", + embedder.name, + vector.len() + ); + } + + // A second call must not renegotiate the width. + let again = embedder + .model + .embed_query("gamma") + .await + .unwrap_or_else(|e| panic!("{}: embed_query failed: {e}", embedder.name)); + assert_eq!( + again.len(), + declared, + "{}: the width changed between calls", + embedder.name + ); + assert!( + embedder.model.signature().contains(&declared.to_string()), + "{}: the signature should pin the dimensionality: {}", + embedder.name, + embedder.model.signature() + ); + } +} + +/// A batch must return one vector per input, **in input order**. +/// +/// Asserted by comparing each batched vector against the same text embedded on +/// its own. A backend that reorders or drops a row would still return the right +/// *count*, so counting alone cannot catch it — and a silent reorder poisons an +/// index in a way that only shows up later as bad retrieval. +#[tokio::test] +async fn local_embedders_return_one_vector_per_input_in_order() { + let texts: Vec = [ + "the cat sat on the mat", + "quarterly revenue exceeded expectations", + "rust is a systems programming language", + ] + .iter() + .map(|s| s.to_string()) + .collect(); + + for embedder in reachable_embedders().await { + let batched = embedder + .model + .embed(&texts) + .await + .unwrap_or_else(|e| panic!("{}: batched embed failed: {e}", embedder.name)); + + assert_eq!( + batched.len(), + texts.len(), + "{}: expected one vector per input", + embedder.name + ); + + for (index, text) in texts.iter().enumerate() { + let alone = embedder + .model + .embed_query(text) + .await + .unwrap_or_else(|e| panic!("{}: single embed failed: {e}", embedder.name)); + + // Not asserted bit-identical: batching can change the arithmetic + // (padding, batch-size-dependent kernels) by a hair. Alignment is + // what matters, and a misplaced row scores nowhere near 1.0. + let alignment = cosine_similarity(&batched[index], &alone); + assert!( + alignment > 0.99, + "{}: batched vector {index} does not match `{text}` embedded alone \ + (cosine {alignment:.4}) — the batch is misaligned", + embedder.name + ); + } + } +} + +/// The vectors must carry meaning: a paraphrase has to sit closer to a sentence +/// than an unrelated sentence does. +/// +/// Without this every other assertion here would still pass for a backend that +/// returned constant or random vectors of the right shape. +#[tokio::test] +async fn local_embedders_place_paraphrases_closer_than_unrelated_text() { + for embedder in reachable_embedders().await { + let vectors = embedder + .model + .embed(&[ + "a small dog is barking loudly in the garden".to_string(), + "the little puppy is making a lot of noise outside".to_string(), + "compile times regressed after the dependency upgrade".to_string(), + ]) + .await + .unwrap_or_else(|e| panic!("{}: embed failed: {e}", embedder.name)); + + let paraphrase = cosine_similarity(&vectors[0], &vectors[1]); + let unrelated = cosine_similarity(&vectors[0], &vectors[2]); + + assert!( + paraphrase > unrelated, + "{}: a paraphrase ({paraphrase:.4}) should score above unrelated text \ + ({unrelated:.4}) — the embedding space carries no meaning", + embedder.name + ); + } +} + +/// The full retrieval path: index real documents, query with words that appear +/// in **none** of them, and get the topically right document first. +/// +/// The query deliberately shares no content words with the target document, so +/// nothing here can be satisfied by lexical overlap or by the exact-text +/// shortcut that makes the mock-backed test tautological. +#[tokio::test] +async fn local_embedders_rank_the_right_document_first() { + for embedder in reachable_embedders().await { + let retriever = Retriever::new( + Arc::clone(&embedder.model), + Arc::new(InMemoryVectorStore::new()), + ); + + retriever + .index(vec![ + ( + "animals".into(), + "Cats purr when they are content and knead soft blankets with their paws." + .into(), + json!({ "topic": "animals" }), + ), + ( + "finance".into(), + "The central bank raised interest rates to curb accelerating inflation.".into(), + json!({ "topic": "finance" }), + ), + ( + "programming".into(), + "Ownership and borrowing let the compiler prove memory safety without a \ + garbage collector." + .into(), + json!({ "topic": "programming" }), + ), + ]) + .await + .unwrap_or_else(|e| panic!("{}: indexing failed: {e}", embedder.name)); + + for (query, expected) in [ + ("Why do felines make a rumbling sound?", "animals"), + ("monetary policy and rising prices", "finance"), + ("how does the language avoid a GC?", "programming"), + ] { + let hits = retriever + .retrieve(query, 3) + .await + .unwrap_or_else(|e| panic!("{}: retrieval failed: {e}", embedder.name)); + + assert_eq!( + hits.len(), + 3, + "{}: expected all 3 documents back", + embedder.name + ); + assert_eq!( + hits[0].id, + expected, + "{}: `{query}` should rank `{expected}` first, got `{}` \ + (scores: {:?})", + embedder.name, + hits[0].id, + hits.iter() + .map(|h| (h.id.as_str(), h.score)) + .collect::>() + ); + assert_eq!( + hits[0].metadata["topic"], expected, + "{}: metadata should survive the round trip", + embedder.name + ); + } + } +} + +/// Blank input is handled **differently by the two adapters**, and this pins +/// both so the divergence is visible rather than discovered in production. +/// +/// A corpus routinely contains empty documents, and the danger is a backend +/// that silently *drops* them: the remaining vectors shift up by one, every id +/// after the blank binds to the wrong vector, and the index is permanently +/// corrupted in a way that only surfaces later as bad retrieval. Neither +/// adapter does that — but they avoid it in opposite ways: +/// +/// - [`OllamaEmbeddingModel`] returns an **empty vector per blank input**, +/// preserving positions, and never dials the server. +/// - [`OpenAiEmbeddingModel`] **rejects the batch** with a validation error +/// naming the offending index, because the hosted OpenAI endpoint rejects +/// empty strings outright. +/// +/// Both are position-safe. They are not, however, interchangeable: code that +/// indexes a corpus containing blanks works against Ollama and fails against +/// any OpenAI-compatible endpoint, including a local LM Studio. Callers must +/// filter blanks themselves rather than rely on either behaviour. +#[tokio::test] +async fn blank_input_is_position_safe_on_both_adapters() { + let blanks = [" ".to_string(), "\n".to_string()]; + + for embedder in reachable_embedders().await { + match embedder.model.embed(&blanks).await { + Ok(vectors) => { + assert_eq!( + embedder.name, "ollama", + "only the Ollama adapter is expected to accept an all-blank batch" + ); + assert_eq!( + vectors.len(), + blanks.len(), + "ollama: an all-blank batch must still return one slot per input" + ); + assert!( + vectors.iter().all(Vec::is_empty), + "ollama: blank inputs should yield empty vectors, not fabricated ones" + ); + } + Err(error) => { + let error = error.to_string(); + assert_eq!( + embedder.name, "lmstudio", + "only the OpenAI-compatible adapter is expected to reject blanks" + ); + // Rejecting is acceptable; rejecting without saying which input + // is at fault would leave the caller unable to fix their corpus. + assert!( + error.contains("index 0"), + "lmstudio: the rejection should name the offending index, got: {error}" + ); + } + } + } +} + +/// A missing model must fail with a message that says how to fix it. +/// +/// "model not found" is the single most common local-embedding failure — the +/// operator simply has not pulled it — so the error is expected to name the +/// remedy rather than surface a bare 404. +#[tokio::test] +async fn ollama_reports_a_missing_embedding_model_with_remediation() { + if reachable_embedders() + .await + .iter() + .all(|embedder| embedder.name != "ollama") + { + return; + } + + let base_url = env_or("LOCAL_OLLAMA_EMBED_URL", "http://localhost:11434"); + let missing = OllamaEmbeddingModel::new(&base_url, "definitely-not-installed-abc123", 768); + + let error = missing + .embed(&["hello".to_string()]) + .await + .expect_err("an uninstalled model must not silently succeed") + .to_string(); + + assert!( + error.contains("ollama pull definitely-not-installed-abc123"), + "the error should tell the operator how to install it, got: {error}" + ); +} + +// --------------------------------------------------------------------------- +// Offline unit coverage +// +// These run on every `cargo test` with no network, pinning the configuration +// rules that the live tests above depend on. +// --------------------------------------------------------------------------- + +/// [`OllamaEmbeddingModel`] takes the **server root**, not an API endpoint. +/// +/// This is easy to get wrong because the *chat* side of the same server is +/// configured with a `/v1` suffix, and Ollama's embedding API is not under +/// `/v1` at all. Pointing the adapter at `/v1` or `/api` yields a 404 on every +/// call, so it is rejected at construction with a message naming the mistake. +#[test] +fn the_ollama_embedding_adapter_rejects_an_endpoint_url() { + for bad in [ + "http://localhost:11434/v1", + "http://localhost:11434/api", + "http://localhost:11434/v1/embeddings", + ] { + let error = OllamaEmbeddingModel::try_new(bad, "nomic-embed-text", 768) + .expect_err(&format!("{bad} should be rejected")) + .to_string(); + assert!( + error.contains("server root"), + "{bad} should be rejected with a message naming the fix, got: {error}" + ); + } + + assert!( + OllamaEmbeddingModel::try_new("http://localhost:11434", "nomic-embed-text", 768).is_ok() + ); +} + +/// The OpenAI embedding adapter must be usable against a local server: no +/// credential, and no OpenAI-specific `dimensions` parameter on the wire. +#[test] +fn the_openai_embedding_adapter_can_be_pointed_at_a_local_server() { + let model = OpenAiEmbeddingModel::new("") + .with_base_url("http://localhost:1234/v1") + .with_model("text-embedding-nomic-embed-text-v1.5") + .with_required_api_key(false) + .with_send_dimensions(false) + .with_dimensions(768); + + assert_eq!(model.dimensions(), 768); + assert_eq!(model.model_id(), "text-embedding-nomic-embed-text-v1.5"); + // The signature partitions persisted vectors, so a local model must not + // collide with a hosted OpenAI one of the same width. + assert!( + model + .signature() + .contains("text-embedding-nomic-embed-text-v1.5") + ); +} + +/// Blank-only batches short-circuit without dialling, so the positional +/// guarantee holds even with no server running. +#[tokio::test] +async fn blank_batches_are_answered_without_a_server() { + let model = OllamaEmbeddingModel::new("http://127.0.0.1:9", "nomic-embed-text", 768); + let vectors = model + .embed(&["".to_string(), " ".to_string()]) + .await + .expect("an all-blank batch never reaches the network"); + assert_eq!(vectors, vec![Vec::::new(), Vec::new()]); +} diff --git a/tests/live_local_models.rs b/tests/live_local_models.rs new file mode 100644 index 0000000..e11d68e --- /dev/null +++ b/tests/live_local_models.rs @@ -0,0 +1,819 @@ +//! LIVE local-runtime coverage: Ollama and LM Studio, through the same +//! [`OpenAiModel`] adapter that serves every hosted provider. +//! +//! [`live_provider_matrix`] already proves that a *configured* endpoint answers +//! chat, streaming, and a one-shot tool call. That is necessary but not +//! sufficient for a local runtime, because the ways local servers break are +//! specific to them and mostly invisible to a single-call probe: +//! +//! - they reject request shapes the hosted APIs accept (a named `tool_choice` +//! object, `response_format: {"type": "json_object"}`) with an HTTP 400, +//! - they send **no** `Authorization` header and 401 on an unexpected one, +//! - they serve whatever model the operator loaded, under an id no preset can +//! guess, and +//! - a small quantised model can emit one syntactically valid tool call and +//! still fail to *use the result*, which is what an agent actually needs. +//! +//! So this file drives the whole loop rather than one call: discovery → +//! chat → streaming → one-shot tool call → **a full tool round trip through +//! [`AgentHarness`]** (model asks for the tool, the harness runs it, the model +//! consumes the result and answers) → structured JSON output. +//! +//! Every test runs against every runtime that is reachable, so one file covers +//! both Ollama and LM Studio and any future OpenAI-compatible local server. +//! +//! # Configuration +//! +//! | Variable | Default | +//! |---|---| +//! | `LOCAL_MODEL_TESTS` | unset — **required** to dial anything | +//! | `LOCAL_OLLAMA_BASE_URL` | `http://localhost:11434/v1` | +//! | `LOCAL_OLLAMA_MODEL` | first tool-capable id `GET /v1/models` advertises | +//! | `LOCAL_LMSTUDIO_BASE_URL` | `http://localhost:1234/v1` | +//! | `LOCAL_LMSTUDIO_MODEL` | first non-embedding id `GET /v1/models` advertises | +//! +//! # Skips gracefully +//! +//! Dialling is opt-in via `LOCAL_MODEL_TESTS=1`, so a bare `cargo test` never +//! starts a multi-second local inference run. Set it and any runtime that is +//! not listening — or that has no usable model loaded — is reported and +//! skipped rather than failed, so a box running only Ollama still passes. +//! +//! # Run +//! +//! ```text +//! LOCAL_MODEL_TESTS=1 cargo test --test live_local_models -- --nocapture +//! ``` + +use std::sync::Arc; +use std::sync::Mutex; + +use async_trait::async_trait; +use futures::StreamExt; +use serde_json::{Value, json}; + +use tinyagents::Result; +use tinyagents::harness::context::{RunConfig, RunContext}; +use tinyagents::harness::message::Message; +use tinyagents::harness::model::{ + ChatModel, ModelRequest, ModelStreamItem, ResponseFormat, StreamAccumulator, ToolChoice, +}; +use tinyagents::harness::providers::openai::OpenAiModel; +use tinyagents::harness::providers::{ProviderKind, ProviderSpec}; +use tinyagents::harness::runtime::{AgentHarness, InvalidArgsPolicy, RunPolicy}; +use tinyagents::harness::testkit::{EventRecorder, Trajectory}; +use tinyagents::harness::tool::{Tool, ToolCall, ToolResult, ToolSchema}; + +/// Per-call ceiling. A cold local model has to load several GB off disk before +/// it emits its first token, so this is deliberately generous. +const TIMEOUT_MS: u64 = 180_000; + +/// Token budget for every call here, sized for a **reasoning** model. +/// +/// This is not padding. Local reasoning models (`qwen3` in both Ollama and LM +/// Studio) spend tokens on a hidden reasoning channel *before* emitting a +/// single visible character, and that channel draws from the same `max_tokens` +/// budget. Asking `qwen3-4b` for one word with `max_tokens: 32` returns +/// `finish_reason: "length"`, 30 reasoning tokens, and **empty** visible +/// content — a completion that looks like a provider bug and is really a budget +/// that never reached the answer. +/// +/// The crate already treats this as a first-class failure mode — see +/// [`RunPolicy::empty_response_retries`], whose documentation names `qwen3` via +/// Ollama specifically — so the tests must not reintroduce it by being frugal. +/// A budget this size is what a host talking to local models should use. +/// +/// [`RunPolicy::empty_response_retries`]: tinyagents::harness::runtime::RunPolicy::empty_response_retries +const MAX_TOKENS: u32 = 1024; + +/// The weather the fake tool always reports. Distinctive enough that finding it +/// in the model's final answer proves the tool *result* reached the model, +/// rather than the model inventing a plausible temperature. +/// +/// Both sentinels are chosen to survive paraphrase, which a live assertion +/// against a 3B model must account for. The temperature is positive because a +/// leading minus sign is routinely dropped when the model restates the value, +/// and the condition is matched on its stem (`sandstorm`) because the model +/// freely re-inflects the word it was given (`hailing` came back as `hail`). +const SENTINEL_TEMPERATURE: &str = "41"; +const SENTINEL_CONDITION: &str = "sandstorms"; +/// The substring actually searched for in the model's prose. +const SENTINEL_CONDITION_STEM: &str = "sandstorm"; + +// --------------------------------------------------------------------------- +// Runtime discovery +// --------------------------------------------------------------------------- + +/// One local runtime under test. +struct LocalRuntime { + /// Display name used in skip/failure messages. + name: &'static str, + kind: ProviderKind, + base_url: String, + model: String, +} + +impl LocalRuntime { + /// Builds the adapter for this runtime. + /// + /// Local runtimes need no credential, so the key is a placeholder that + /// `from_spec` never puts on the wire: the [`ProviderKind`] presets carry + /// `requires_api_key: false`, which selects `AuthStyle::None`. + fn model(&self) -> OpenAiModel { + let spec = ProviderSpec::for_kind(self.kind.clone()) + .with_base_url(&self.base_url) + .with_model(&self.model); + OpenAiModel::from_spec(spec, "local").expect("local runtime spec is complete") + } +} + +/// Ids that are embedding-only or otherwise not chat models. LM Studio and +/// Ollama both list embedding models alongside chat models in `/v1/models`, +/// and asking one to chat is an error that has nothing to do with the harness. +fn is_embedding_id(id: &str) -> bool { + let id = id.to_ascii_lowercase(); + id.contains("embed") || id.contains("bge") || id.contains("rerank") +} + +/// Chat model ids that are known **not** to support tool calling, so the +/// tool-loop tests would fail on the model rather than on the code under test. +fn is_tool_incapable_id(id: &str) -> bool { + let id = id.to_ascii_lowercase(); + // Base/text-completion and vision-only tags in the common local catalogues. + id.contains("-base") || id.contains("stable-diffusion") || id.contains("whisper") +} + +/// Resolves one runtime, or returns the reason it cannot be tested. +/// +/// Model choice is explicit (`*_MODEL`) or discovered from the server: a local +/// runtime serves whatever the operator loaded, so no default id can be +/// hard-coded without 404ing on most installs. +async fn discover( + name: &'static str, + kind: ProviderKind, + url_var: &str, + model_var: &str, + default_url: &str, +) -> std::result::Result { + let base_url = std::env::var(url_var) + .ok() + .filter(|v| !v.trim().is_empty()) + .unwrap_or_else(|| default_url.to_string()); + + // A model id is needed to construct the adapter at all, so probe with a + // placeholder purely to reach `list_models`, which is model-independent. + let probe_spec = ProviderSpec::for_kind(kind.clone()) + .with_base_url(&base_url) + .with_model("probe"); + let probe = OpenAiModel::from_spec(probe_spec, "local") + .map_err(|e| format!("{name}: invalid configuration: {e}"))?; + + let listed = probe + .list_models() + .await + .map_err(|e| format!("{name}: not reachable at {base_url} ({e})"))?; + + if let Some(explicit) = std::env::var(model_var) + .ok() + .filter(|v| !v.trim().is_empty()) + { + return Ok(LocalRuntime { + name, + kind, + base_url, + model: explicit.trim().to_string(), + }); + } + + let model = listed + .iter() + .map(|entry| entry.id.clone()) + .find(|id| !is_embedding_id(id) && !is_tool_incapable_id(id)) + .ok_or_else(|| { + format!( + "{name}: reachable at {base_url} but serves no chat model \ + (saw {} id(s)); load one or set {model_var}", + listed.len() + ) + })?; + + Ok(LocalRuntime { + name, + kind, + base_url, + model, + }) +} + +/// Every local runtime that is reachable and usable right now. +/// +/// Returns an empty vector (after explaining why, on stderr) when the opt-in +/// switch is unset or nothing is listening, which is what lets these tests pass +/// on a machine with no local runtime at all. +async fn reachable_runtimes() -> Vec { + if std::env::var("LOCAL_MODEL_TESTS") + .ok() + .filter(|v| !v.trim().is_empty() && v != "0") + .is_none() + { + eprintln!( + "skipping live local-model tests: set LOCAL_MODEL_TESTS=1 to dial local runtimes \ + (LOCAL_MODEL_TESTS=1 cargo test --test live_local_models -- --nocapture)" + ); + return Vec::new(); + } + + let candidates = [ + ( + "ollama", + ProviderKind::Ollama, + "LOCAL_OLLAMA_BASE_URL", + "LOCAL_OLLAMA_MODEL", + "http://localhost:11434/v1", + ), + ( + "lmstudio", + ProviderKind::LmStudio, + "LOCAL_LMSTUDIO_BASE_URL", + "LOCAL_LMSTUDIO_MODEL", + "http://localhost:1234/v1", + ), + ]; + + let mut ready = Vec::new(); + for (name, kind, url_var, model_var, default_url) in candidates { + match discover(name, kind, url_var, model_var, default_url).await { + Ok(runtime) => { + eprintln!( + "local runtime `{}` ready: {} @ {}", + runtime.name, runtime.model, runtime.base_url + ); + ready.push(runtime); + } + Err(reason) => eprintln!("skipping {reason}"), + } + } + // Opting in explicitly and then reaching nothing must not look like success. + // Every assertion in this file is inside a `for` over this list, so an empty + // list makes the whole suite pass while testing nothing at all — the exact + // failure mode these tests exist to rule out. + assert!( + !ready.is_empty(), + "LOCAL_MODEL_TESTS is set but no local runtime is reachable. Start Ollama \ + (`ollama serve` + `ollama pull llama3.2:3b`) or LM Studio (`lms server start` + \ + load a model), or unset LOCAL_MODEL_TESTS to skip." + ); + ready +} + +/// The [`RunPolicy`] a host should use to drive a small local model. +/// +/// The crate default is [`InvalidArgsPolicy::Fail`], which aborts the entire +/// run the first time a model calls a registered tool with schema-invalid +/// arguments. That is a reasonable default for a frontier model, where the case +/// is nearly always a genuine bug — but a 3B quantised model omits a required +/// argument often enough that `Fail` makes the loop unusably brittle: one bad +/// call and the run dies rather than the model getting a chance to correct +/// itself. [`InvalidArgsPolicy::NormalizeThenReturnToolError`] repairs the +/// common provider-shape defects and otherwise hands the validation error back +/// to the model as a tool result, and the recovery still consumes a tool-call +/// budget slot so the loop stays bounded. +/// +/// This is observed behaviour, not a hypothetical: with the default policy, +/// `llama3.2:3b` fails this file's tool-loop test with +/// `tool `get_weather` arguments.city is required`. +fn local_run_policy() -> RunPolicy { + RunPolicy { + invalid_args: InvalidArgsPolicy::NormalizeThenReturnToolError, + ..RunPolicy::default() + } +} + +fn base_request(messages: Vec) -> ModelRequest { + ModelRequest { + messages, + max_tokens: Some(MAX_TOKENS), + timeout_ms: Some(TIMEOUT_MS), + ..ModelRequest::default() + } +} + +/// How many times a tool-calling assertion may re-roll the model. +/// +/// `ToolChoice::Required` is a **request**, not a guarantee, and a 3B model +/// declines it often enough to matter: measured over 12 consecutive +/// `tool_choice: "required"` calls, `llama3.2:3b` via Ollama returned a +/// structured tool call 11 times and plain prose once. Sometimes it answers the +/// question from parametric memory instead. +/// +/// That is a property of the model, not of the code under test, so asserting it +/// per-call would make this suite a coin flip. These tests ask the question that +/// is actually about our code — *can this runtime, through this adapter, +/// produce a well-formed tool call and consume its result?* — and a bounded +/// re-roll answers exactly that: a runtime that genuinely cannot do it fails all +/// `TOOL_ATTEMPTS` times, while one that can will not fail all of them. +/// +/// Any host driving a local model needs the same allowance. A tool call is not +/// something you can assume happened — check, and retry. +const TOOL_ATTEMPTS: usize = 4; + +/// Runs `attempt` up to [`TOOL_ATTEMPTS`] times, returning once one succeeds. +/// +/// Panics with the final attempt's message when every try fails, so a genuine +/// breakage still surfaces the real error rather than a bare count. +async fn with_tool_reroll(runtime: &LocalRuntime, label: &str, mut attempt: F) +where + F: FnMut() -> Fut, + Fut: std::future::Future>, +{ + let mut last = String::new(); + for round in 1..=TOOL_ATTEMPTS { + match attempt().await { + Ok(()) => { + if round > 1 { + eprintln!( + " {}: {label} succeeded on attempt {round}/{TOOL_ATTEMPTS} \ + (the model declined the tool on earlier tries)", + runtime.name + ); + } + return; + } + Err(error) => last = error, + } + } + panic!( + "{}: {label} failed all {TOOL_ATTEMPTS} attempts against `{}`; last error: {last}", + runtime.name, runtime.model + ); +} + +// --------------------------------------------------------------------------- +// A real tool, with a real schema, that records what it was asked +// --------------------------------------------------------------------------- + +/// A weather tool that returns a fixed, deliberately implausible reading. +/// +/// Unlike [`FakeTool`](tinyagents::harness::testkit::FakeTool) this declares a +/// real JSON Schema with a required argument, which is the thing a small local +/// model actually has to get right. +struct WeatherTool { + /// Every `city` argument the model supplied, in call order. + seen: Mutex>, +} + +impl WeatherTool { + fn new() -> Self { + Self { + seen: Mutex::new(Vec::new()), + } + } + + fn cities(&self) -> Vec { + self.seen.lock().expect("weather tool lock").clone() + } + + fn schema_json() -> Value { + json!({ + "type": "object", + "properties": { + "city": { "type": "string", "description": "City name, e.g. \"Paris\"." } + }, + "required": ["city"] + }) + } +} + +#[async_trait] +impl Tool<()> for WeatherTool { + fn name(&self) -> &str { + "get_weather" + } + + fn description(&self) -> &str { + "Returns the current weather for a given city. Always use this tool for weather questions." + } + + fn schema(&self) -> ToolSchema { + ToolSchema::new(self.name(), self.description(), Self::schema_json()) + } + + async fn call(&self, _state: &(), call: ToolCall) -> Result { + let city = call + .arguments + .get("city") + .and_then(Value::as_str) + .unwrap_or_default() + .to_string(); + self.seen.lock().expect("weather tool lock").push(city); + Ok(ToolResult::text( + call.id, + "get_weather", + format!("{SENTINEL_TEMPERATURE} degrees Celsius and {SENTINEL_CONDITION}",), + )) + } +} + +/// Forces [`ToolChoice::Required`] on the **first** model call of a run, then +/// gets out of the way. +/// +/// Without this the tool-loop test is really two tests wearing one coat: does +/// the 3B model *choose* to call the tool, and does the harness then feed the +/// result back correctly. The first is model judgment and is genuinely +/// stochastic — `llama3.2:3b` answers the weather question from parametric +/// memory roughly one run in six no matter how the system prompt is worded. +/// Only the second is a property of the code under test, so the choice is +/// forced and the loop behaviour is what gets asserted. +/// +/// The force must not persist: with `Required` on every call the model can +/// never emit a final answer and the run would spin until it hit the tool-call +/// cap. +struct ForceFirstToolCall { + calls: Mutex, +} + +impl ForceFirstToolCall { + fn new() -> Self { + Self { + calls: Mutex::new(0), + } + } +} + +#[async_trait] +impl tinyagents::harness::middleware::Middleware<(), ()> for ForceFirstToolCall { + fn name(&self) -> &str { + "force_first_tool_call" + } + + async fn before_model( + &self, + _ctx: &mut RunContext<()>, + _state: &(), + request: &mut ModelRequest, + ) -> Result<()> { + let mut calls = self.calls.lock().expect("force-first-tool-call lock"); + if *calls == 0 { + request.tool_choice = ToolChoice::Required; + } + *calls += 1; + Ok(()) + } +} + +/// The same tool as a bare schema, for the one-shot (harness-free) probe. +fn weather_schema() -> ToolSchema { + ToolSchema::new( + "get_weather", + "Returns the current weather for a given city.", + WeatherTool::schema_json(), + ) +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +/// `GET /v1/models` must answer, and advertise at least one usable id. +/// +/// This is the discovery path a host uses to populate a model picker, and it is +/// the only way to learn a local runtime's model ids — there is no catalogue to +/// hard-code. +#[tokio::test] +async fn local_runtimes_advertise_their_loaded_models() { + for runtime in reachable_runtimes().await { + let listed = runtime + .model() + .list_models() + .await + .unwrap_or_else(|e| panic!("{}: list_models failed: {e}", runtime.name)); + + assert!( + !listed.is_empty(), + "{}: a reachable runtime should advertise at least one model", + runtime.name + ); + assert!( + listed.iter().any(|entry| entry.id == runtime.model), + "{}: the selected model `{}` should appear in its own /models listing", + runtime.name, + runtime.model + ); + } +} + +/// A single-turn chat call must return non-empty assistant text. +#[tokio::test] +async fn local_runtimes_answer_a_single_turn_chat() { + for runtime in reachable_runtimes().await { + let response = runtime + .model() + .invoke( + &(), + base_request(vec![Message::user( + "Reply with exactly the single word: hello", + )]), + ) + .await + .unwrap_or_else(|e| panic!("{}: chat failed: {e}", runtime.name)); + + assert!( + !response.text().trim().is_empty(), + "{}: chat returned empty assistant text", + runtime.name + ); + } +} + +/// Streaming must produce genuinely incremental deltas, not one final chunk. +/// +/// A local server that buffers the whole completion and emits it as a single +/// SSE event technically "streams" but breaks every incremental consumer, so +/// the delta count is asserted, not just the merged text. +#[tokio::test] +async fn local_runtimes_stream_incremental_deltas() { + for runtime in reachable_runtimes().await { + let mut stream = runtime + .model() + .stream( + &(), + base_request(vec![Message::user("Count from one to five, in words.")]), + ) + .await + .unwrap_or_else(|e| panic!("{}: stream failed to start: {e}", runtime.name)); + + let mut deltas = 0usize; + let mut accumulator = StreamAccumulator::new(); + while let Some(item) = stream.next().await { + if matches!(item, ModelStreamItem::MessageDelta(_)) { + deltas += 1; + } + accumulator.push(&item); + } + let response = accumulator + .finish() + .unwrap_or_else(|e| panic!("{}: stream produced no response: {e}", runtime.name)); + + assert!( + deltas > 1, + "{}: expected incremental deltas, got {deltas}", + runtime.name + ); + assert!( + !response.text().trim().is_empty(), + "{}: streamed text was empty", + runtime.name + ); + } +} + +/// The model must emit one well-formed tool call with parseable arguments. +/// +/// `ToolChoice::Required` is used deliberately: local servers historically +/// reject a *named* tool choice object, and the transport degrades that shape +/// for local runtimes. This asserts the degradation actually works end to end. +#[tokio::test] +async fn local_runtimes_emit_a_parseable_tool_call() { + for runtime in reachable_runtimes().await { + let model = runtime.model(); + with_tool_reroll(&runtime, "one-shot tool call", || async { + let mut request = base_request(vec![Message::user( + "What is the weather in Paris right now? Call the get_weather tool.", + )]); + request.tools = vec![weather_schema()]; + request.tool_choice = ToolChoice::Required; + + let response = model + .invoke(&(), request) + .await + .map_err(|e| format!("forced tool call failed: {e}"))?; + + let call = response + .message + .tool_calls + .first() + .ok_or_else(|| "no tool call returned".to_string())?; + + if call.name != "get_weather" { + return Err(format!("called {:?}, expected get_weather", call.name)); + } + if let Some(invalid) = &call.invalid { + return Err(format!("tool arguments did not parse: {invalid}")); + } + // The requested city must have reached us, but *where* in the + // arguments is a model-quality question this layer cannot fix. + // Small models bury it under a wrapper key or echo the tool's own + // JSON Schema with the value filled in — all captured shapes are + // listed on `unwrap_wrapped_arguments`. Recovering them is the + // harness's job and is asserted by + // `local_runtimes_complete_a_full_tool_loop`; the provider + // adapter's own contract is only that it surfaced a well-formed + // call carrying the argument somewhere. + let rendered = call.arguments.to_string(); + if !rendered.contains("Paris") { + return Err(format!("tool call carried no `city` argument: {rendered}")); + } + Ok(()) + }) + .await; + } +} + +/// The full agent loop: the model asks for a tool, the harness runs it, and the +/// model answers **using the result**. +/// +/// This is the test that a one-shot tool probe cannot replace. Emitting a tool +/// call is easy; correctly consuming a `tool` role message and producing a +/// grounded final answer is where small quantised local models — and any bug in +/// how the adapter serialises tool results back onto the wire — actually break. +#[tokio::test] +async fn local_runtimes_complete_a_full_tool_loop() { + for runtime in reachable_runtimes().await { + with_tool_reroll(&runtime, "full tool loop", || async { + let tool = Arc::new(WeatherTool::new()); + + let mut harness: AgentHarness<()> = AgentHarness::new(); + harness.register_tool(tool.clone()); + harness + .register_model("local", Arc::new(runtime.model())) + .set_default_model("local") + .with_policy(local_run_policy()) + .push_middleware(Arc::new(ForceFirstToolCall::new())); + + let recorder = EventRecorder::new(); + let ctx = RunContext::new(RunConfig::new("live-local-tool-loop"), ()) + .with_events(recorder.sink()); + + let run = harness + .invoke_in_context( + &(), + ctx, + vec![ + Message::system( + "You answer weather questions. You must call the get_weather tool \ + to obtain the weather, then state the temperature and condition it \ + returned. Never invent weather data.", + ), + Message::user("What is the weather in Paris right now?"), + ], + ) + .await + .map_err(|e| format!("tool-loop run failed: {e}"))?; + + let traj = Trajectory::from_events(recorder.events()); + if !traj.completed() { + return Err("the run did not reach completion".to_string()); + } + if !traj.tool_was_called("get_weather") { + return Err("the model never called get_weather".to_string()); + } + + let cities = tool.cities(); + if !cities.iter().any(|c| c.to_lowercase().contains("paris")) { + return Err(format!( + "the tool was not asked about Paris, got {cities:?}" + )); + } + + // Two model calls minimum: one to request the tool, one to consume + // its result. A single call would mean the loop never fed the + // result back — the whole point of this test. + if run.model_calls < 2 { + return Err(format!( + "expected at least 2 model calls (request + consume), got {}", + run.model_calls + )); + } + + let final_text = run.text().unwrap_or_default(); + if final_text.trim().is_empty() { + return Err("the loop produced no final answer".to_string()); + } + // The answer must be grounded in what the tool returned rather than + // in what the model already believes about the weather in Paris. + let grounded = final_text.contains(SENTINEL_TEMPERATURE) + || final_text.to_lowercase().contains(SENTINEL_CONDITION_STEM); + if !grounded { + return Err(format!( + "the final answer ignored the tool result (expected \ + {SENTINEL_TEMPERATURE} or {SENTINEL_CONDITION_STEM}): {final_text}" + )); + } + Ok(()) + }) + .await; + } +} + +/// Structured JSON output must come back as parseable JSON. +/// +/// Local servers reject `response_format: {"type": "json_object"}` with a 400 +/// and want a `json_schema` instead; the transport degrades that shape for +/// local runtimes, and this asserts the degraded request is both accepted and +/// honoured. +#[tokio::test] +async fn local_runtimes_produce_structured_json_output() { + for runtime in reachable_runtimes().await { + let mut request = base_request(vec![Message::user( + "Paris is the capital of France and has about 2.1 million residents. \ + Return it as JSON with keys `city` and `country`.", + )]); + request.response_format = Some(ResponseFormat::JsonObject); + + let response = runtime + .model() + .invoke(&(), request) + .await + .unwrap_or_else(|e| panic!("{}: structured output call failed: {e}", runtime.name)); + + let text = response.text(); + let parsed: Value = serde_json::from_str(text.trim()).unwrap_or_else(|e| { + panic!( + "{}: json_object response did not parse as JSON ({e}): {text}", + runtime.name + ) + }); + assert!( + parsed.is_object(), + "{}: expected a JSON object, got {parsed}", + runtime.name + ); + } +} + +// --------------------------------------------------------------------------- +// Offline unit coverage +// +// These run on every `cargo test` with no network and no local server, so the +// discovery/classification rules stay pinned even when the live tests skip. +// --------------------------------------------------------------------------- + +#[test] +fn embedding_ids_are_excluded_from_chat_model_discovery() { + for id in [ + "nomic-embed-text:latest", + "text-embedding-nomic-embed-text-v1.5", + "bge-m3", + "BAAI/bge-reranker-v2-m3", + ] { + assert!( + is_embedding_id(id), + "{id} should be treated as embedding-only" + ); + } + for id in ["llama3.2:3b", "qwen3-4b", "gpt-oss-20b"] { + assert!(!is_embedding_id(id), "{id} is a chat model"); + } +} + +#[test] +fn tool_incapable_ids_are_excluded_from_chat_model_discovery() { + assert!(is_tool_incapable_id("llama-3.2-3b-base")); + assert!(is_tool_incapable_id("whisper-large-v3")); + assert!(!is_tool_incapable_id("qwen3-4b")); +} + +#[test] +fn local_presets_need_no_credential_and_default_to_their_own_ports() { + let ollama = ProviderSpec::for_kind(ProviderKind::Ollama); + assert!(!ollama.requires_api_key); + assert_eq!(ollama.base_url, "http://localhost:11434/v1"); + + let lmstudio = ProviderSpec::for_kind(ProviderKind::LmStudio); + assert!(!lmstudio.requires_api_key); + assert_eq!(lmstudio.base_url, "http://localhost:1234/v1"); + // LM Studio serves whatever GGUF is loaded, so there is no default id to + // guess; callers must supply one. + assert!( + lmstudio.model.is_empty(), + "the LM Studio preset must not guess a model id" + ); +} + +/// Pins the default that makes local tool loops brittle, and the opt-in that +/// fixes them. +/// +/// If the crate default ever changes to a recovering policy, this test fails +/// and [`local_run_policy`]'s rationale (and the docs pointing hosts at it) +/// should be revisited rather than the assertion simply flipped. +#[test] +fn invalid_tool_arguments_abort_the_run_unless_recovery_is_opted_into() { + assert_eq!( + RunPolicy::default().invalid_args, + InvalidArgsPolicy::Fail, + "the crate default aborts a run on schema-invalid tool arguments" + ); + assert_eq!( + local_run_policy().invalid_args, + InvalidArgsPolicy::NormalizeThenReturnToolError, + "local runtimes need the recovering policy so a small model can self-correct" + ); +} + +#[test] +fn a_local_runtime_spec_builds_without_an_api_key() { + let spec = ProviderSpec::for_kind(ProviderKind::LmStudio).with_model("qwen3-4b"); + let model = OpenAiModel::from_spec(spec, "local").expect("local spec builds"); + assert_eq!(model.base_url(), "http://localhost:1234/v1"); + assert_eq!(model.model(), "qwen3-4b"); +}