642d5a256791
tests.rs
//! End-to-end tests: a tiny std TcpListener HTTP server stands in for the model APIs, and a real
//! engine runs the tools. No real network.
use super::*;
use craft_anything_platform::sync::Mutex;
use serde_json::{Value, json};
use std::io::{BufRead, BufReader, Read, Write};
use std::net::TcpListener;
/// #1096, #1239: an AI's timeout_secs is its own limit on a streamed answer, starting included;
/// unset or 0 leaves the defaults.
#[test]
fn an_ais_own_timeout_bounds_its_streamed_answer() {
let with = |timeout_secs| BackendOptions { timeout_secs, ..Default::default() };
assert_eq!(stream_limit(&with(None)), None);
assert_eq!(stream_limit(&with(Some(0))), None);
assert_eq!(stream_limit(&with(Some(2700))), Some(std::time::Duration::from_secs(2700)));
}
/// #1225: with no session id saved (a backend added before there was one, or found from an
/// environment key), OpenCode Go still gets its session header, the same on every request.
#[test]
fn opencode_go_gets_a_session_header_without_a_saved_id() {
let (h, _d) = hub();
assert!(h.config().install_id.is_none(), "nothing saved");
let id = h.session_id();
assert!(!id.is_empty());
assert_eq!(h.session_id(), id, "the same for every request of this run");
let client = compatible_client("OpenCode Go", openai::OPENCODE_GO_BASE, Some("sk-test".into()), "m", &id);
assert!(client.extra_headers.iter().any(|(name, value)| *name == openai::OPENCODE_GO_SESSION_HEADER && *value == id), "{:?}", client.extra_headers);
}
/// #1225: a saved OpenCode Go backend with no id (one from an older version, or a copied config)
/// gets one at startup, once, and keeps it after a restart. A hub with no Go backend makes none.
#[test]
fn a_saved_opencode_go_backend_gets_one_stable_session_id_at_startup() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("config.toml");
let keys = || KeyStore::file_only(dir.path().join("ai-keys.toml"));
// A hub whose session starts from the settings file, as the app and the CLI do.
let started = |path: &std::path::Path| {
let mut session = craft_anything_engine::testing::session();
session.settings = craft_anything_engine::Settings::load_from(path);
AiHub::with_stores(Arc::new(Engine::new(session)), path.to_path_buf(), keys())
};
let saved_id = || AiConfig::from_table(&craft_anything_engine::Settings::load_from(&path).ai).unwrap().install_id;
// A Go backend saved the way an older version did: no id in the file.
let first = started(&path);
add(&first, "OpenCode Go (subscription)", BackendKind::OpenaiCompatible, openai::OPENCODE_GO_BASE, Some("sk-test"));
let kept = std::fs::read_to_string(&path).unwrap();
std::fs::write(&path, kept.lines().filter(|l| !l.trim_start().starts_with("install_id")).collect::<Vec<_>>().join("\n")).unwrap();
assert!(saved_id().is_none(), "the test starts without an id");
let startup = started(&path);
assert!(startup.ensure_session_id().unwrap(), "one is made and saved");
let id = saved_id().expect("saved to the settings file");
assert!(id.starts_with("craft-"), "{id}");
assert!(!startup.ensure_session_id().unwrap(), "made once, not again on the next start");
assert_eq!(saved_id().as_deref(), Some(id.as_str()));
assert_eq!(started(&path).session_id(), id, "the same id after a restart");
// No Go backend, no id.
let other_dir = tempfile::tempdir().unwrap();
let other_path = other_dir.path().join("config.toml");
let plain = AiHub::with_stores(
Arc::new(Engine::new(craft_anything_engine::testing::session())),
other_path.clone(),
KeyStore::file_only(other_dir.path().join("ai-keys.toml")),
);
add(&plain, "Claude", BackendKind::Anthropic, "https://api.anthropic.com", Some("sk-test"));
assert!(!plain.ensure_session_id().unwrap());
assert!(AiConfig::from_table(&craft_anything_engine::Settings::load_from(&other_path).ai).unwrap().install_id.is_none());
}
/// The model list lives at /v1 on the service root, not under the chat path's /openai/v1.
#[test]
fn opencode_go_lists_models_at_the_documented_url() {
assert_eq!(format!("{}/models", opencode_go_models_base(openai::OPENCODE_GO_BASE)), "https://opencode.ai/inference/go/v1/models");
assert_eq!(opencode_go_models_base("https://opencode.ai/inference/go/openai/v1/"), "https://opencode.ai/inference/go/v1");
assert_eq!(opencode_go_models_base("http://127.0.0.1:11434/v1"), "http://127.0.0.1:11434/v1");
}
#[derive(Debug, Clone)]
struct Seen {
path: String,
headers: Vec<(String, String)>,
body: Value,
}
impl Seen {
fn header(&self, k: &str) -> Option<&str> {
self.headers.iter().find(|(n, _)| n.eq_ignore_ascii_case(k)).map(|(_, v)| v.as_str())
}
}
/// Serve canned responses `(status, content_type, body)` in order; record requests.
fn mock(responses: Vec<(u16, &'static str, String)>) -> (String, Arc<Mutex<Vec<Seen>>>) {
let l = TcpListener::bind("127.0.0.1:0").unwrap();
let url = format!("http://{}", l.local_addr().unwrap());
let seen = Arc::new(Mutex::new(Vec::new()));
let s2 = seen.clone();
std::thread::spawn(move || {
let mut responses = responses.into_iter().peekable();
while let Some((next_status, next_type, _)) = responses.peek() {
let lists_next = *next_status == 200 && *next_type == "application/json";
let Ok((stream, _)) = l.accept() else { return };
let mut r = BufReader::new(stream.try_clone().unwrap());
let mut line = String::new();
r.read_line(&mut line).unwrap();
let path = line.split_whitespace().nth(1).unwrap_or_default().to_string();
// Scripted questions are model lists answered with a list. Any other question is a
// window probe (crate::window) the test didn't script: say nothing, as a server that
// doesn't report its window does, and keep the scripted answer for the request after.
if line.starts_with("GET ") && !(lists_next && path.ends_with("/models")) {
let mut w = stream;
write!(w, "HTTP/1.1 404 Not Found\r\ncontent-length: 0\r\nconnection: close\r\n\r\n").unwrap();
w.flush().unwrap();
continue;
}
let Some((status, ctype, reply)) = responses.next() else { return };
let mut headers = Vec::new();
let mut len = 0usize;
loop {
let mut h = String::new();
r.read_line(&mut h).unwrap();
let h = h.trim_end().to_string();
if h.is_empty() {
break;
}
if let Some((k, v)) = h.split_once(':') {
if k.eq_ignore_ascii_case("content-length") {
len = v.trim().parse().unwrap();
}
headers.push((k.trim().to_string(), v.trim().to_string()));
}
}
let mut buf = vec![0; len];
r.read_exact(&mut buf).unwrap();
let body = serde_json::from_slice(&buf).unwrap_or(Value::Null);
s2.lock().unwrap().push(Seen { path, headers, body });
let mut w = stream;
let reason = if status == 200 { "OK" } else { "Error" };
write!(w, "HTTP/1.1 {status} {reason}\r\ncontent-type: {ctype}\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{reply}", reply.len()).unwrap();
w.flush().unwrap();
}
});
(url, seen)
}
/// A stand-in server on a free port that answers every request (and any number of them, unlike
/// [`mock`]) with `status` and `body`; its `/v1` address.
pub(crate) fn serve(status: &'static str, body: &'static str) -> String {
let l = TcpListener::bind("127.0.0.1:0").unwrap();
let base = format!("http://{}/v1", l.local_addr().unwrap());
std::thread::spawn(move || {
for mut s in l.incoming().flatten() {
let (mut got, mut buf) = (Vec::new(), [0u8; 4096]);
while !got.windows(4).any(|w| w == b"\r\n\r\n") {
match s.read(&mut buf) {
Ok(0) | Err(_) => break,
Ok(n) => got.extend_from_slice(&buf[..n]),
}
}
if !got.is_empty() {
let _ = write!(s, "HTTP/1.1 {status}\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}", body.len());
}
}
});
base
}
/// #48: a coding agent's test only runs `--version`, and its report says so.
#[test]
fn a_coding_agent_test_says_it_only_checked_the_version() {
let (h, _d) = hub();
let config = BackendConfig { name: "Agent".into(), kind: BackendKind::AgentCli, model: "opencode".into(), base_url: None, ssh_host: None, enabled: true };
// cargo stands in for the agent's program: it's there wherever these tests run.
let options = BackendOptions { command: Some("cargo".into()), ..Default::default() };
h.add_backend_with(BackendEntry { config, options }, None).unwrap();
let r = h.test_backend("Agent").unwrap();
assert!(r.version_only && r.reply.starts_with("cargo "), "{r:?}");
assert_eq!(serde_json::to_value(&r).unwrap()["version_only"], true, "the CLI's ai test prints it");
assert_eq!((r.program.as_deref(), r.saved_program_replaced), (Some("cargo"), None), "the saved program ran");
}
#[cfg(unix)]
#[test]
/// The selected Codex CLI checks sign-in without a prompt or exposing raw status output.
fn codex_connection_test_checks_login_without_sending_a_prompt() {
use std::os::unix::fs::PermissionsExt;
let (h, _d) = hub();
let dir = tempfile::tempdir().unwrap();
let program = dir.path().join("codex");
let signed_in = dir.path().join("connected");
let script = format!(
"#!/bin/sh\nif [ \"$1\" = --version ]; then echo codex-test; exit 0; fi\nif [ \"$1\" = login ] && [ \"$2\" = status ]; then\n if [ -f '{}' ]; then echo 'Logged in using ChatGPT LOGIN_STATUS_TEST_ONLY' >&2; exit 0; fi\n echo 'Not logged in' >&2; exit 1\nfi\nexit 99\n",
signed_in.display()
);
std::fs::write(&program, script).unwrap();
std::fs::set_permissions(&program, std::fs::Permissions::from_mode(0o700)).unwrap();
let config = BackendConfig { name: "Codex test".into(), kind: BackendKind::AgentCli, model: "codex".into(), base_url: None, ssh_host: None, enabled: true };
let options = BackendOptions { command: Some(program.display().to_string()), ..Default::default() };
h.add_backend_with(BackendEntry { config, options }, None).unwrap();
let error = h.test_backend("Codex test").unwrap_err();
assert!(error.message.contains("not signed in"));
assert!(error.remediation.unwrap().contains("Sign in with ChatGPT"));
std::fs::write(signed_in, b"test marker, not a credential").unwrap();
let result = h.test_backend("Codex test").unwrap();
assert!(result.version_only, "login is not a model/quota test");
assert!(result.reply.contains("signed in"));
assert!(!serde_json::to_string(&result).unwrap().contains("LOGIN_STATUS_TEST_ONLY"), "raw login-status output never enters the report");
}
/// A second adapter supplies sign-in results through the same API, respecting its config folder.
#[cfg(unix)]
#[test]
fn local_agent_connection_checks_use_the_adapter_and_its_selected_folder() {
use std::os::unix::fs::PermissionsExt;
let (h, _d) = hub();
let dir = tempfile::tempdir().unwrap();
let program = dir.path().join("agent");
std::fs::write(
&program,
r#"#!/bin/sh
if [ "$1" = --version ]; then echo agent-test; exit 0; fi
if [ "$1" = auth ] && [ "$2" = status ] && [ "$3" = --json ]; then
cat "$CLAUDE_CONFIG_DIR/status.json"; exit 0
fi
exit 99
"#,
)
.unwrap();
std::fs::set_permissions(&program, std::fs::Permissions::from_mode(0o700)).unwrap();
let config =
BackendConfig { name: "Other agent".into(), kind: BackendKind::AgentCli, model: "claude-code".into(), base_url: None, ssh_host: None, enabled: true };
let options = BackendOptions { command: Some(program.display().to_string()), config_dir: Some(dir.path().display().to_string()), ..Default::default() };
h.add_backend_with(BackendEntry { config, options }, None).unwrap();
let status = dir.path().join("status.json");
std::fs::write(&status, r#"{"loggedIn":false}"#).unwrap();
let error = h.test_backend("Other agent").unwrap_err();
assert_eq!(error.message, "Other agent is installed but is not signed in.");
assert!(error.remediation.is_some());
std::fs::write(&status, r#"{"loggedIn":true,"private":"LOGIN_STATUS_TEST_ONLY"}"#).unwrap();
let result = h.test_backend("Other agent").unwrap();
assert!(result.version_only && result.reply.contains("signed in"));
assert!(!serde_json::to_string(&result).unwrap().contains("LOGIN_STATUS_TEST_ONLY"));
for unknown in ["not json", "{}"] {
std::fs::write(&status, unknown).unwrap();
assert_eq!(h.test_backend("Other agent").unwrap().reply, "agent-test", "unknown sign-in leaves the version reply unchanged");
}
}
/// #721: a saved path an update removed. A test that passes through the usual program says so, and
/// doctor doesn't blame PATH when nothing can be found.
#[test]
fn a_saved_program_that_is_gone_is_named_by_test_and_doctor() {
let (h, _d) = hub();
let dir = tempfile::tempdir().unwrap();
let gone = dir.path().join("old-build").join("agent").display().to_string();
let backend = |name: &str, model: &str| BackendEntry {
config: BackendConfig { name: name.into(), kind: BackendKind::AgentCli, model: model.into(), base_url: None, ssh_host: None, enabled: true },
options: BackendOptions { command: Some(gone.clone()), ..Default::default() },
};
// cargo stands in for the agent's usual program: it's there wherever these tests run.
h.add_backend_with(backend("Usual", "cargo"), None).unwrap();
h.add_backend_with(backend("Nowhere", "no-such-agent-program"), None).unwrap();
let r = h.test_backend("Usual").unwrap();
assert_eq!((r.program.as_deref(), r.saved_program_replaced.as_deref()), (Some("cargo"), Some(gone.as_str())));
let reach = h.check_reachability();
let nowhere = reach.iter().find(|(name, _)| name == "Nowhere").map(|(_, r)| r.clone()).unwrap();
assert_eq!(nowhere, Err(format!("there's no program at {gone}")));
// What Settings says: each AI whose saved program is gone, and what runs instead. One whose
// saved program is there, that has none saved, or that has nothing to run instead (doctor
// says that one), isn't listed.
let there = dir.path().join("agent-here");
std::fs::write(&there, "").unwrap();
let mut kept = backend("Kept", "cargo");
kept.options.command = Some(there.display().to_string());
h.add_backend_with(kept, None).unwrap();
let mut usual = backend("Unsaved", "cargo");
usual.options.command = None;
h.add_backend_with(usual, None).unwrap();
let replaced = h.replaced_programs();
assert_eq!(replaced.get("Usual").map(String::as_str), Some("cargo"));
assert!(!replaced.contains_key("Nowhere") && !replaced.contains_key("Kept") && !replaced.contains_key("Unsaved"), "{replaced:?}");
}
/// Every known agent with an adapter of its own is offered when adding a coding agent.
#[test]
fn every_agent_with_its_own_adapter_is_offered() {
let adapters = cli_agents::adapters::AgentAdapters::built_in();
let own: Vec<&str> = cli_agents::KNOWN.iter().map(|(id, _, _)| *id).filter(|id| adapters.for_agent(id).ids().contains(id)).collect();
assert!(own.contains(&"maki"), "{own:?}");
for id in own {
assert!(BackendKind::AgentCli.suggested_models().contains(&id), "{id} isn't offered");
}
}
/// #37: doctor asks a local server for its models instead of trusting an open port.
#[test]
fn a_local_server_is_reachable_only_when_it_lists_models() {
let (h, _d) = hub();
add(&h, "Web page", BackendKind::OpenaiCompatible, &serve("404 Not Found", "404 page not found"), None);
add(&h, "Real", BackendKind::OpenaiCompatible, &serve("200 OK", r#"{"data":[{"id":"qwen3-8b"}]}"#), None);
let reach = h.check_reachability();
let of = |n: &str| reach.iter().find(|(name, _)| name == n).map(|(_, r)| r.clone()).unwrap();
assert!(of("Web page").is_err_and(|e| e.contains("404 page not found")), "{reach:?}");
assert!(of("Real").is_ok(), "{reach:?}");
}
fn sse(events: &[Value]) -> String {
events.iter().map(|e| format!("event: {}\r\ndata: {}\r\n\r\n", e["type"].as_str().unwrap_or("message"), e)).collect()
}
fn chat_sse(chunks: &[Value]) -> String {
let mut s: String = chunks.iter().map(|c| format!("data: {c}\n\n")).collect();
s.push_str("data: [DONE]\n\n");
s
}
fn hub() -> (AiHub, tempfile::TempDir) {
let dir = tempfile::tempdir().unwrap();
let engine = Arc::new(Engine::new(craft_anything_engine::testing::session()));
let h = AiHub::with_stores(engine, dir.path().join("config.toml"), KeyStore::file_only(dir.path().join("ai-keys.toml")));
(h, dir)
}
/// A named backend stays off for chat, including before API or agent setup can run.
#[test]
fn disabled_backends_are_refused_when_chat_starts() {
let (h, _dir) = hub();
for kind in [BackendKind::OpenaiCompatible, BackendKind::AgentCli] {
// A malformed tunnel port fails before opening SSH if API setup is reached.
let (base_url, ssh_host) =
if kind == BackendKind::OpenaiCompatible { ("http://127.0.0.1:not-a-port/v1", Some("unused".into())) } else { ("http://127.0.0.1:0/v1", None) };
h.add_backend(
BackendConfig { name: "Off".into(), kind, model: "no-such-agent".into(), base_url: Some(base_url.into()), ssh_host, enabled: false },
None,
)
.unwrap();
let error = h.start(Some("Off"), false).err().expect("chat must refuse a disabled backend");
assert_eq!(error.code, ErrorCode::BadSettings);
assert!(error.message.contains("Off") && error.message.contains("turned off"), "{error:?}");
assert!(error.remediation.as_deref().is_some_and(|s| s.contains("Settings → AI")), "{error:?}");
}
}
/// R2.9: an API backend's kept transcript is resumed only in the mode that saved it.
#[test]
fn an_api_transcript_is_resumed_only_in_its_mode() {
let (h, dir) = hub();
h.add_backend(
BackendConfig {
name: "Local".into(),
kind: BackendKind::OpenaiCompatible,
model: "test-model".into(),
base_url: Some("http://127.0.0.1:0/v1".into()),
ssh_host: None,
enabled: true,
},
None,
)
.unwrap();
let entry = h.entry("Local").unwrap();
let path = dir.path().join("transcript.json");
let history = vec![crate::model::Msg::user_text("a Deep conversation")];
crate::context::save(&path, &history, true).unwrap();
assert!(h.start_builder(&entry, false, Some(path.clone()), true, None).unwrap().resumed(), "Deep continues a Deep transcript");
assert!(!h.start_builder(&entry, false, Some(path.clone()), false, None).unwrap().resumed(), "Clean room starts fresh");
crate::context::save(&path, &history, false).unwrap();
assert!(!h.start_builder(&entry, false, Some(path), true, None).unwrap().resumed(), "Deep doesn't continue a Clean room transcript");
}
/// Compare checks either position; enabled named and automatic chat choices still work.
#[test]
fn disabled_backends_are_refused_on_either_side_of_compare() {
let (h, _dir) = hub();
for (name, enabled) in [("Off", false), ("On", true), ("Other", true)] {
h.add_backend(
BackendConfig {
name: name.into(),
kind: BackendKind::OpenaiCompatible,
model: "test-model".into(),
base_url: Some("http://127.0.0.1:0/v1".into()),
ssh_host: None,
enabled,
},
None,
)
.unwrap();
}
for (left, right) in [("Off", "On"), ("On", "Off")] {
let error = h.compare(left, right, true).err().expect("compare must refuse a disabled backend");
assert_eq!(error.code, ErrorCode::BadSettings);
assert!(error.message.contains("Off") && error.message.contains("turned off"), "{error:?}");
}
assert!(h.start(Some("On"), false).is_ok());
assert!(h.start(None, false).is_ok());
assert!(h.compare("On", "Other", false).is_ok());
let unknown = h.start(Some("Missing"), false).err().unwrap();
assert!(unknown.message.contains("No AI backend named"), "{unknown:?}");
}
/// A CLI stand-in checks saved selections, and an explicit one runs natively with our MCP server.
#[cfg(unix)]
#[test]
fn opencode_selection_is_validated_saved_and_run_natively_with_our_mcp_server() {
use std::os::unix::fs::PermissionsExt;
let (h, dir) = hub();
let program = dir.path().join("agent");
let catalogue = dir.path().join("models.json");
let seen = dir.path().join("arguments.txt");
let quote = |p: &std::path::Path| ssh::shell_quote(&p.display().to_string());
let script = format!(
"#!/bin/sh\ncase \"$1\" in\napi) cat {};;\nrun) if [ \"$2\" = '--help' ]; then printf '%s\\n' '--format --standalone'; exit 0; fi; printf '%s\\n' \"$@\" > {}; cat \"$OPENCODE_CONFIG\" >> {}; printf 'done';;\n*) exit 8;;\nesac\n",
quote(&catalogue),
quote(&seen),
quote(&seen)
);
std::fs::write(&program, script).unwrap();
std::fs::set_permissions(&program, std::fs::Permissions::from_mode(0o700)).unwrap();
let model = json!({"id":"alias","providerID":"test","name":"Test","enabled":true,"type":"chat","variants":[{"id":"high"},{"id":"max"}]});
std::fs::write(&catalogue, json!({"data":[model]}).to_string()).unwrap();
let cfg = BackendConfig { name: "OpenCode".into(), kind: BackendKind::AgentCli, model: "opencode".into(), base_url: None, ssh_host: None, enabled: true };
h.add_backend_with(
BackendEntry { config: cfg, options: BackendOptions { command: Some(program.display().to_string()), acp: Some(true), ..Default::default() } },
None,
)
.unwrap();
assert_eq!(h.model_choices("OpenCode").unwrap(), ["test/alias"]);
assert!(h.set_agent_model("OpenCode", Some("test/alias"), Some(Effort::Max)).unwrap());
let saved = AiConfig::from_settings(&Settings::load_from(&dir.path().join("config.toml")));
assert_eq!(saved.get("OpenCode").unwrap().options.cli_thinking, Some(Effort::Max));
assert_eq!(saved.get("OpenCode").unwrap().config.model, "opencode");
let mut conversation = h.start(Some("OpenCode"), false).unwrap();
conversation.send("test prompt");
let deadline = Instant::now() + Duration::from_secs(5);
let mut events = Vec::new();
while conversation.is_busy() && Instant::now() < deadline {
events.extend(conversation.poll());
std::thread::sleep(Duration::from_millis(10));
}
events.extend(conversation.poll());
events.retain(|event| !matches!(event, AgentEvent::Status { .. }));
assert!(matches!(events.last(), Some(AgentEvent::TurnDone { .. })), "{events:?}");
// The arguments, one per line, then the config file the run was handed.
let ran = std::fs::read_to_string(&seen).unwrap();
let args: Vec<&str> = ran.lines().collect();
assert_eq!(args.get(..4), Some(["run", "--standalone", "--model", "test/alias#max"].as_slice()), "{ran}");
let config: Value = serde_json::from_str(ran.lines().find(|l| l.starts_with("{\"mcp\"")).unwrap()).unwrap();
assert_eq!(config["mcp"][cli_agents::MCP_NAME]["type"], "local", "the run gets our MCP server: {ran}");
assert!(h.set_agent_model("OpenCode", Some("test/alias"), Some(Effort::Xhigh)).is_err());
assert_eq!(h.options()["OpenCode"].cli_thinking, Some(Effort::Max), "a failed validation changes nothing");
let stale = h.model_context("OpenCode").unwrap();
h.set_enabled("OpenCode", false).unwrap();
assert!(h.set_agent_selection(&stale, None, None).is_err(), "a late picker cannot overwrite newer settings");
std::fs::write(&catalogue, r#"{"data":[]}"#).unwrap();
assert!(h.set_model("OpenCode", "test/alias").is_err(), "removed models are not offered as a fallback");
h.set_agent_model("OpenCode", None, None).unwrap();
assert!(h.options()["OpenCode"].cli_model.is_none());
assert!(h.options()["OpenCode"].cli_thinking.is_none());
}
/// Contention is an error, while an available session retains its actual optional project and remains saveable.
#[test]
fn model_context_distinguishes_a_busy_session_from_no_project() {
use craft_anything_engine::project::{Project, Purpose};
let (h, dir) = hub();
let backend =
BackendConfig { name: "OpenCode".into(), kind: BackendKind::AgentCli, model: "opencode".into(), base_url: None, ssh_host: None, enabled: true };
h.add_backend(backend.clone(), None).unwrap();
assert_eq!(h.model_context("OpenCode").unwrap().directory, None);
let root = dir.path().join("test.cany");
{
let mut session = h.engine.session();
session.open_project(Project::create(&root, "test", Purpose::Education).unwrap()).unwrap();
assert!(h.model_context("OpenCode").is_err(), "lock contention must not capture a missing project");
}
let context = h.model_context("OpenCode").unwrap();
assert_eq!(context.directory.as_ref(), Some(&root));
assert_eq!(context.backend.config, backend);
assert!(h.set_agent_selection(&context, None, None).unwrap(), "a capture after contention must remain saveable");
assert!(h.model_context("missing").is_err());
}
#[test]
fn a_failed_save_changes_nothing() {
let dir = tempfile::tempdir().unwrap();
// The settings "file" can't be written: its folder is a file.
std::fs::write(dir.path().join("blocked"), "").unwrap();
let engine = Arc::new(Engine::new(craft_anything_engine::testing::session()));
let h = AiHub::with_stores(engine, dir.path().join("blocked").join("config.toml"), KeyStore::file_only(dir.path().join("ai-keys.toml")));
let cfg = BackendConfig { name: "Kimi".into(), kind: BackendKind::Openrouter, model: "m".into(), base_url: None, ssh_host: None, enabled: true };
assert!(h.add_backend(cfg, None).is_err());
assert!(h.backends().is_empty(), "the app shows only what was saved");
}
/// #261: one bad price made the whole [ai] section read as empty, and adding a backend saved
/// that over the backends, roles and prices.
#[test]
fn ai_settings_that_cant_be_read_are_never_saved_over() {
let (h, d) = hub();
add(&h, "Claude Code", BackendKind::AgentCli, "http://127.0.0.1:9", None);
add(&h, "Kimi K3.1 Pro (OpenRouter)", BackendKind::Openrouter, "http://127.0.0.1:9", None);
h.set_role(Role::Builder, Some("Kimi K3.1 Pro (OpenRouter)")).unwrap();
let file = d.path().join("config.toml");
let mut bad = h.engine.session().settings.clone();
let price: toml::Table = toml::from_str(
"[m]
input_per_mtok = 1.0
",
)
.unwrap();
bad.ai.insert("prices".into(), toml::Value::Table(price));
bad.save_to(&file).unwrap();
h.engine.session().settings = bad;
let before = std::fs::read(&file).unwrap();
let cfg = BackendConfig { name: "Other".into(), kind: BackendKind::Openrouter, model: "m".into(), base_url: None, ssh_host: None, enabled: true };
let e = h.add_backend(cfg, Some("sk-test".into())).unwrap_err();
assert_eq!(e.code, ErrorCode::BadSettings, "{e:?}");
assert!(h.set_role(Role::Builder, None).is_err());
assert!(h.set_enabled("Claude Code", false).is_err());
assert!(h.remove_backend("Claude Code").is_err());
let options = crate::model_options::SetModelOptions { backend: "Claude Code".into(), expected_cli_model: None, context: None };
assert_eq!(h.set_model_options(&options).unwrap_err().code, ErrorCode::BadSettings);
assert_eq!(std::fs::read(&file).unwrap(), before, "the settings file was changed");
assert!(h.keys.get("Other").is_none(), "a refused add stored its key");
}
#[test]
fn the_person_chooses_who_builds() {
let (h, _d) = hub();
add(&h, "Claude Code", BackendKind::AgentCli, "http://127.0.0.1:9", None);
add(&h, "Kimi K3.1 Pro (OpenRouter)", BackendKind::Openrouter, "http://127.0.0.1:9", Some("sk-test"));
assert_eq!(h.chosen(Role::Builder), None, "automatic until chosen");
assert!(h.set_role(Role::Builder, Some("Nobody")).is_err(), "only a configured backend");
h.set_role(Role::Builder, Some("Kimi K3.1 Pro (OpenRouter)")).unwrap();
assert_eq!(h.chosen(Role::Builder).as_deref(), Some("Kimi K3.1 Pro (OpenRouter)"));
// The extractor is chosen the same way, and has no default.
assert_eq!(h.chosen(Role::Extractor), None);
h.set_role(Role::Extractor, Some("Kimi K3.1 Pro (OpenRouter)")).unwrap();
assert_eq!(h.chosen(Role::Extractor).as_deref(), Some("Kimi K3.1 Pro (OpenRouter)"));
// A removed backend can't stay the builder.
h.remove_backend("Kimi K3.1 Pro (OpenRouter)").unwrap();
assert_eq!((h.chosen(Role::Builder), h.chosen(Role::Extractor)), (None, None));
h.set_role(Role::Builder, Some("Claude Code")).unwrap();
h.set_role(Role::Builder, None).unwrap();
assert_eq!(h.chosen(Role::Builder), None, "back to automatic");
}
/// #53: an agent app installed where it isn't looked for gets its program set (and cleared).
#[test]
fn an_agent_apps_program_is_set_and_cleared() {
let (h, _d) = hub();
add(&h, "OpenCode", BackendKind::AgentCli, "http://127.0.0.1:9", None);
let program = |h: &AiHub| h.options().get("OpenCode").and_then(|o| o.command.clone());
assert_eq!(program(&h), None);
assert!(h.set_command("OpenCode", Some(" /opt/oc/bin/opencode ")).unwrap());
assert_eq!(program(&h).as_deref(), Some("/opt/oc/bin/opencode"));
assert!(!h.set_command("Nobody", Some("/x")).unwrap(), "only a configured backend");
assert!(h.set_command("OpenCode", Some(" ")).unwrap());
assert_eq!(program(&h), None, "blank finds it by its usual name again");
}
/// #467: a folder or app bundle can't be an agent's program (the file picker returns
/// `Antigravity.app` on a Mac); it's refused and the saved program stays as it was.
#[test]
fn an_app_bundle_is_refused_as_an_agents_program() {
let (h, d) = hub();
add(&h, "OpenCode", BackendKind::AgentCli, "http://127.0.0.1:9", None);
let program = |h: &AiHub| h.options().get("OpenCode").and_then(|o| o.command.clone());
let bundle = d.path().join("Antigravity.app");
std::fs::create_dir_all(bundle.join("Contents")).unwrap();
let err = h.set_command("OpenCode", Some(&bundle.display().to_string())).unwrap_err();
assert_eq!(err.code, ErrorCode::BadInput, "{err:?}");
assert!(err.message.contains("is a folder"), "{}", err.message);
assert_eq!(program(&h), None, "nothing is saved for a refused folder");
// A file, and a bare name, are still accepted.
let file = d.path().join("agy");
std::fs::write(&file, "#!/bin/sh\n").unwrap();
assert!(h.set_command("OpenCode", Some(&file.display().to_string())).unwrap());
assert!(h.set_command("OpenCode", Some("agy")).unwrap());
assert_eq!(program(&h).as_deref(), Some("agy"));
}
/// #423: a saved AI that is switched off isn't set up, so a copy found on the computer is offered
/// again (to be turned on); one that is on is left out.
#[test]
fn a_switched_off_agent_is_found_again() {
let (h, _d) = hub();
add(&h, "Claude Code", BackendKind::AgentCli, "http://127.0.0.1:9", None);
let saved = h.backends().into_iter().next().unwrap();
let found = |name: &str| Detected { suggestion: BackendConfig { name: name.into(), ..saved.clone() }, reason: "Found".into(), needs_key: false };
let offered =
|h: &AiHub| leave_out_set_up(vec![found("Claude Code"), found("Codex")], &h.backends()).into_iter().map(|d| d.suggestion.name).collect::<Vec<_>>();
assert_eq!(offered(&h), ["Codex"], "on: Claude Code is set up, so it isn't offered");
h.set_enabled("Claude Code", false).unwrap();
assert_eq!(offered(&h), ["Claude Code", "Codex"], "off: it's found again, to be turned on");
assert!(!is_on(&h.backends(), "Claude Code") && !is_on(&h.backends(), "Codex"));
}
/// Claude Code's folder (`CLAUDE_CONFIG_DIR`) is chosen per AI, so two can be two accounts; only
/// a full path, and only for an agent app that keeps one.
#[test]
fn claude_codes_folder_is_set_per_ai() {
let (h, _d) = hub();
let claude = |name: &str, ssh: Option<&str>| BackendConfig {
name: name.into(),
kind: BackendKind::AgentCli,
model: "claude-code".into(),
base_url: None,
ssh_host: ssh.map(str::to_string),
enabled: true,
};
h.add_backend(claude("Claude Code", None), None).unwrap();
h.add_backend(claude("Claude Code 2", None), None).unwrap();
h.add_backend(claude("Claude on box", Some("box")), None).unwrap();
h.add_backend(BackendConfig { model: "codex".into(), ..claude("Codex", None) }, None).unwrap();
let folder = |h: &AiHub, name: &str| h.options().get(name).and_then(|o| o.config_dir.clone());
assert!(h.set_config_dir("Claude Code 2", Some(" ~/.claude-work ")).unwrap());
assert_eq!((folder(&h, "Claude Code"), folder(&h, "Claude Code 2").as_deref()), (None, Some("~/.claude-work")));
let here = if cfg!(windows) { r"C:\Claude\work" } else { "/Users/someone/Claude work" };
assert!(h.set_config_dir("Claude Code", Some(here)).unwrap());
assert_eq!(folder(&h, "Claude Code").as_deref(), Some(here));
assert!(h.set_config_dir("Claude on box", Some("/home/someone/.claude-b")).unwrap(), "a path on that computer");
assert!(h.set_config_dir("Claude Code", Some("claude-work")).unwrap_err().message.contains("isn't a full path"));
assert!(h.set_config_dir("Codex", Some("/x")).unwrap_err().message.contains("doesn't keep a folder"));
assert!(h.set_config_dir("Codex", None).unwrap(), "clearing is always fine");
assert!(!h.set_config_dir("Nobody", Some("/x")).unwrap(), "only a configured backend");
assert!(h.set_config_dir("Claude Code 2", Some(" ")).unwrap());
assert_eq!(folder(&h, "Claude Code 2"), None, "blank is its usual folder again");
assert_eq!(folder(&h, "Claude Code").as_deref(), Some(here), "each AI keeps its own");
assert_eq!(crate::folder_var(&claude("x", None)), Some("CLAUDE_CONFIG_DIR"));
}
/// #123, #127: the model an AI uses is changed; Claude Code is offered Claude's models.
#[test]
fn an_ais_model_is_changed() {
let (h, _d) = hub();
let url = serve("200 OK", r#"{"object":"list","data":[{"id":"nomic-embed-text"},{"id":"qwen3:8b"}]}"#);
h.add_server("Box", &url, Some("nomic-embed-text"), None).unwrap();
assert_eq!(h.model_choices("Box").unwrap(), ["nomic-embed-text", "qwen3:8b"], "the server's own list");
assert!(h.set_model("Box", " qwen3:8b ").unwrap());
assert_eq!(h.backends().into_iter().find(|b| b.name == "Box").unwrap().model, "qwen3:8b");
let claude =
BackendConfig { name: "Claude Code".into(), kind: BackendKind::AgentCli, model: "claude-code".into(), base_url: None, ssh_host: None, enabled: true };
h.add_backend(claude, None).unwrap();
assert_eq!(
h.model_choices("Claude Code").unwrap(),
[crate::anthropic::CLAUDE_CODE_DEFAULT, "claude-opus-5-5", "claude-sonnet-5-5", "claude-haiku-5-5", "claude-fable-5-1"]
);
assert!(h.set_model("Claude Code", "claude-sonnet-5-5").unwrap());
let claude = h.options().get("Claude Code").and_then(|o| o.cli_model.clone());
assert_eq!(claude.as_deref(), Some("claude-sonnet-5-5"), "the CLI is asked for it; the agent stays Claude Code");
assert_eq!(h.backends().into_iter().find(|b| b.name == "Claude Code").unwrap().model, "claude-code");
// A custom model id the catalogue doesn't list, as a person's Claude Code settings may name.
assert!(h.set_model("Claude Code", "my-custom-model").unwrap());
assert_eq!(h.options().get("Claude Code").and_then(|o| o.cli_model.clone()).as_deref(), Some("my-custom-model"));
// "default" clears the choice: no --model is passed, so Claude Code uses its own settings.
assert!(h.set_model("Claude Code", crate::anthropic::CLAUDE_CODE_DEFAULT).unwrap());
assert_eq!(h.options().get("Claude Code").and_then(|o| o.cli_model.clone()), None);
assert!(!h.set_model("Nobody", "m").unwrap());
assert!(h.set_model("Box", " ").is_err());
// #2109: Codex takes a model the same way, and its default clears it, so no -m is passed.
let codex = BackendConfig { name: "Codex".into(), kind: BackendKind::AgentCli, model: "codex".into(), base_url: None, ssh_host: None, enabled: true };
h.add_backend(codex, None).unwrap();
assert!(h.set_model("Codex", "first").unwrap());
assert_eq!(h.options().get("Codex").and_then(|o| o.cli_model.clone()).as_deref(), Some("first"));
assert!(h.set_model("Codex", crate::anthropic::CLAUDE_CODE_DEFAULT).unwrap());
assert_eq!(h.options().get("Codex").and_then(|o| o.cli_model.clone()), None);
// The Claude Code adapter is also registered as `claude`; the default must clear for that id
// too, or a run would pass `--model default` (#470 review).
let alias = BackendConfig { name: "Claude".into(), kind: BackendKind::AgentCli, model: "claude".into(), base_url: None, ssh_host: None, enabled: true };
h.add_backend(alias, None).unwrap();
assert!(h.set_model("Claude", "my-custom-model").unwrap());
assert_eq!(h.options().get("Claude").and_then(|o| o.cli_model.as_deref()), Some("my-custom-model"));
assert!(h.set_model("Claude", crate::anthropic::CLAUDE_CODE_DEFAULT).unwrap());
assert_eq!(h.options().get("Claude").and_then(|o| o.cli_model.clone()), None);
}
/// Every Claude effort level can be chosen for the Claude API and Claude Code, and only for them.
#[test]
fn a_claude_ais_effort_is_changed() {
let (h, _d) = hub();
let b = |name: &str, kind, model: &str| BackendConfig { name: name.into(), kind, model: model.into(), base_url: None, ssh_host: None, enabled: true };
h.add_backend(b("Claude Code", BackendKind::AgentCli, "claude-code"), None).unwrap();
h.add_backend(b("Codex", BackendKind::AgentCli, "codex"), None).unwrap();
let effort = |h: &AiHub, name: &str| {
let config = h.backends().into_iter().find(|b| b.name == name).unwrap();
crate::effort_in_use(&config, h.options().get(name))
};
assert_eq!(effort(&h, "Claude Code"), Some(Effort::High), "high until it's chosen");
for level in Effort::ALL {
assert!(h.set_effort("Claude Code", Some(level)).unwrap());
assert_eq!(effort(&h, "Claude Code"), Some(level));
}
assert!(h.set_effort("Claude Code", None).unwrap());
assert_eq!(effort(&h, "Claude Code"), Some(Effort::High), "back to the usual level");
assert_eq!(effort(&h, "Codex"), None, "an agent that doesn't take Claude's levels");
assert!(h.set_effort("Codex", Some(Effort::Max)).is_err());
assert!(!h.set_effort("Nobody", Some(Effort::Low)).unwrap());
}
/// #1813: an OpenAI-style API is sent a reasoning level only once the person chooses one, and
/// only one of OpenAI's levels; OpenRouter gets it in its own field.
#[test]
fn an_openai_style_ais_effort_is_sent_only_once_chosen() {
use crate::model::{Msg, Stop, TurnRequest};
let reply = || (200, "text/event-stream", chat_sse(&[json!({"choices":[{"delta":{"content":"hi"},"finish_reason":"stop"}]})]));
for (kind, field) in [(BackendKind::OpenaiCompatible, "/reasoning_effort"), (BackendKind::Openrouter, "/reasoning/effort")] {
let (h, _d) = hub();
let (url, seen) = mock(vec![reply(), reply()]);
let config = BackendConfig { name: "Box".into(), kind, model: "m".into(), base_url: Some(url), ssh_host: None, enabled: true };
h.add_backend(config, Some("k".into())).unwrap();
let ask = |h: &AiHub| {
let mut slot = h.build_model(&h.entry("Box").unwrap()).unwrap();
let messages = [Msg::user_text("hi")];
let request = TurnRequest { system: "S", messages: &messages, tools: &[], effort: prompt::choose_effort(slot.effort), max_tokens: 100 };
slot.model.turn(&request, &mut |_| {}, &Stop::default()).unwrap();
};
let effort = |h: &AiHub| crate::effort_in_use(&h.entry("Box").unwrap().config, h.options().get("Box"));
assert_eq!(effort(&h), None, "nothing is sent until it's chosen");
ask(&h);
assert!(h.set_effort("Box", Some(Effort::Max)).is_err(), "max isn't one of OpenAI's levels");
assert!(h.set_effort("Box", Some(Effort::Minimal)).unwrap());
assert_eq!(effort(&h), Some(Effort::Minimal));
ask(&h);
let seen = seen.lock().unwrap();
assert!(seen[0].body.get("reasoning_effort").is_none() && seen[0].body.get("reasoning").is_none(), "{}", seen[0].body);
assert_eq!(seen[1].body.pointer(field), Some(&json!("minimal")), "{}", seen[1].body);
drop(seen);
assert!(h.set_effort("Box", None).unwrap());
assert_eq!(effort(&h), None);
}
// Gemini is offered only the levels every Gemini model takes.
let gemini = BackendConfig { name: "G".into(), kind: BackendKind::Gemini, model: "gemini-2.5-pro".into(), base_url: None, ssh_host: None, enabled: true };
assert_eq!(crate::effort_levels(&gemini), [Effort::Minimal, Effort::Low, Effort::Medium, Effort::High]);
}
#[test]
fn copilot_uses_shared_selection_and_rejects_failed_discovery_without_saving() {
let (hub, _directory) = hub();
let config =
BackendConfig { name: "GitHub Copilot".into(), kind: BackendKind::AgentCli, model: "copilot".into(), base_url: None, ssh_host: None, enabled: true };
assert!(!config.kind.needs_key());
assert_eq!(model_in_use(&config, None).as_deref(), Some("GitHub Copilot default"));
hub.add_backend_with(
BackendEntry {
config,
options: BackendOptions {
cli_model: Some("chosen-copilot-model".into()),
command: Some("craft-anything-no-such-test-agent".into()),
..Default::default()
},
},
None,
)
.unwrap();
assert!(hub.set_model("GitHub Copilot", "unavailable-model").is_err());
let backend = hub.backends().into_iter().find(|backend| backend.name == "GitHub Copilot").unwrap();
let options = hub.options();
assert_eq!(backend.model, "copilot");
assert_eq!(model_in_use(&backend, options.get(&backend.name)).as_deref(), Some("chosen-copilot-model"));
}
#[test]
fn copilot_context_is_saved_without_changing_shared_thinking() {
use crate::config::CliContext;
let (hub, _directory) = hub();
let config = BackendConfig { name: "Copilot".into(), kind: BackendKind::AgentCli, model: "copilot".into(), base_url: None, ssh_host: None, enabled: true };
hub.add_backend_with(
BackendEntry {
config: config.clone(),
options: BackendOptions { cli_model: Some("test-model".into()), cli_thinking: Some(Effort::Max), ..Default::default() },
},
None,
)
.unwrap();
let params =
crate::model_options::SetModelOptions { backend: "Copilot".into(), expected_cli_model: Some("test-model".into()), context: Some(CliContext::Long) };
hub.set_model_options(¶ms).unwrap();
assert_eq!(hub.options()["Copilot"].cli_context, Some(CliContext::Long));
assert_eq!(hub.options()["Copilot"].cli_thinking, Some(Effort::Max));
hub.add_backend_with(BackendEntry { config, options: BackendOptions { cli_model: Some("another-model".into()), ..Default::default() } }, None).unwrap();
assert_eq!(hub.options()["Copilot"].cli_context, None);
assert_eq!(hub.options()["Copilot"].cli_reasoning_effort, None);
assert!(hub.set_model_options(¶ms).is_err());
}
/// Exercise the installed CLI, not just our argument builder: the loopback model must see
/// exactly the allowed tools in every access mode, including genuinely tool-free questions.
/// Kept opt-in because it needs external programs, never because it needs a real account.
#[test]
#[ignore = "requires Copilot CLI and Node.js; uses an isolated CLI home and a loopback model, never a real account"]
fn copilot_cli_enforces_mcp_only_and_empty_tool_filters() {
use crate::cli_agents::{Access, AcpConversation, AgentAdapter, ArgsCx, McpCommand};
use craft_anything_platform::Process;
/// Only the child's environment changes: no saved CLI credentials, inherited provider key
/// or token helper may reach the fixture. The production adapter still supplies its policy.
struct OfflineCopilot {
url: String,
home: PathBuf,
}
impl AgentAdapter for OfflineCopilot {
fn ids(&self) -> &'static [&'static str] {
&["copilot"]
}
fn args(&self, _context: &ArgsCx) -> Vec<String> {
Vec::new()
}
fn process(&self, program: &str) -> Process {
crate::cli_agents::adapters::Copilot
.process(program)
.env("HOME", &self.home)
.env("COPILOT_HOME", &self.home)
.env("COPILOT_OFFLINE", "true")
.env("COPILOT_PROVIDER_BASE_URL", &self.url)
.env("COPILOT_PROVIDER_TYPE", "openai")
.env("COPILOT_MODEL", "craft-test")
.env_remove("COPILOT_PROVIDER_API_KEY")
.env_remove("COPILOT_PROVIDER_API_KEY_COMMAND")
.env_remove("COPILOT_PROVIDERS_CONFIG")
.env_remove("COPILOT_GITHUB_TOKEN")
.env_remove("GH_TOKEN")
.env_remove("GITHUB_TOKEN")
}
}
for (access, mut expected) in [
(Access::Analyse, vec!["craft-anything-core_target_triage"]),
(Access::Build, vec!["craft-anything-core_target_triage"]),
(Access::CleanRoomBuild, vec!["craft-anything-core_target_triage"]),
(Access::Question, vec!["view", "grep", "glob"]),
(Access::CleanRoomQuestion, Vec::new()),
] {
let directory = tempfile::tempdir().unwrap();
let script = directory.path().join("mcp.cjs");
// Offer one registered tool and one that must be filtered out. Record requests so a
// question that accidentally starts the server is caught even if it calls no tool.
std::fs::write(
&script,
r#"const readline = require('node:readline');
const input = readline.createInterface({input: process.stdin});
input.on('line', line => {
const request = JSON.parse(line);
require('node:fs').appendFileSync(__dirname + '/mcp-requests.log', request.method + '\n');
if (request.id === undefined) return;
let result = {};
if (request.method === 'initialize') {
result = {protocolVersion: '2024-11-05', capabilities: {tools: {}}, serverInfo: {name: 'craft-test', version: '1'}};
} else if (request.method === 'tools/list') {
result = {tools: ['core_target_triage', 'unexpected_fixture_tool'].map(name => ({name, description: name, inputSchema: {type: 'object', properties: {}}}))};
}
process.stdout.write(JSON.stringify({jsonrpc: '2.0', id: request.id, result}) + '\n');
});
"#,
)
.unwrap();
let body = chat_sse(&[
json!({"id":"test", "choices":[{"index":0,"delta":{"role":"assistant","content":"done"},"finish_reason":null}]}),
json!({"id":"test", "choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}),
]);
let (url, seen) = mock(vec![(200, "text/event-stream", body)]);
let program = std::env::var("CRAFT_TEST_COPILOT").unwrap_or_else(|_| "copilot".into());
let mut args = crate::cli_agents::adapters::Copilot.acp_options(&BackendOptions::default(), access);
args.extend(["--acp".into(), "--stdio".into()]);
let mcp = McpCommand { command: "node".into(), args: vec![script.display().to_string()], notes: None };
let question = matches!(access, Access::Question | Access::CleanRoomQuestion);
if !question {
args.extend(crate::cli_agents::adapters::Copilot.acp_mcp_options(&mcp));
}
let mut conversation = AcpConversation::new("Copilot test", "copilot", program, args, directory.path().into(), mcp, None)
.with_adapter(Arc::new(OfflineCopilot { url, home: directory.path().into() }))
.with_question(question);
conversation.send("Reply with done. Do not call any tools.");
let started = Instant::now();
let mut output = Vec::new();
while started.elapsed() < Duration::from_secs(30) && !output.iter().any(|event| matches!(event, AgentEvent::TurnDone { .. })) {
output.extend(conversation.poll());
std::thread::park_timeout(Duration::from_millis(10));
}
assert!(output.iter().any(|event| matches!(event, AgentEvent::TurnDone { .. })), "{output:?}");
assert!(!output.iter().any(|event| matches!(event, AgentEvent::Error { .. })), "{output:?}");
let requests = seen.lock().unwrap();
assert_eq!(requests.len(), 1, "{requests:?}");
// Assert what reached the model, not what the CLI accepted on its command line:
// startup can succeed while a tool is missing or an empty filter restores native tools.
let mut tools: Vec<&str> = requests[0]
.body
.get("tools")
.and_then(Value::as_array)
.into_iter()
.flatten()
.filter_map(|tool| tool.pointer("/function/name").and_then(Value::as_str))
.collect();
tools.sort();
expected.sort();
assert_eq!(tools, expected, "{access:?}");
assert_eq!(directory.path().join("mcp-requests.log").exists(), !question);
}
}
/// A stand-in server that answers `GET <path>` from `routes` and 404 otherwise; its `/v1` address.
pub(crate) fn serve_routes(routes: &'static [(&'static str, &'static str)]) -> String {
let l = TcpListener::bind("127.0.0.1:0").unwrap();
let base = format!("http://{}/v1", l.local_addr().unwrap());
std::thread::spawn(move || {
for mut s in l.incoming().flatten() {
let (mut got, mut buf) = (Vec::new(), [0u8; 4096]);
while !got.windows(4).any(|w| w == b"\r\n\r\n") {
match s.read(&mut buf) {
Ok(0) | Err(_) => break,
Ok(n) => got.extend_from_slice(&buf[..n]),
}
}
let head = String::from_utf8_lossy(&got);
let path = head.split_whitespace().nth(1).unwrap_or_default().to_string();
let (status, body) = routes.iter().find(|(p, _)| *p == path).map_or(("404 Not Found", "not here"), |(_, b)| ("200 OK", *b));
let _ = write!(s, "HTTP/1.1 {status}\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}", body.len());
}
});
base
}
/// #1434: FastFlowLM's `/models` lists every model it can download; taking the first started a
/// 5.5 GB download and unloaded the model in use. A chat model the server says is loaded comes
/// first; without `/api/ps`, or with nothing loaded, the person chooses between several chat
/// models, and the only one is used.
#[test]
fn a_server_added_without_a_model_gets_the_one_it_has_loaded() {
let (h, _d) = hub();
const CATALOGUE: &str = r#"{"object":"list","data":[{"id":"deepseek-r1:8b"},{"id":"gemma3:1b"},{"id":"qwen3.8-mtp-uncensored:27b"}]}"#;
let flm = serve_routes(&[
("/v1/models", CATALOGUE),
("/api/ps", r#"{"models":[{"name":"embed","model":"nomic-embed-text"},{"name":"qwen3.8-mtp-uncensored:27b","model":"qwen3.8-mtp-uncensored:27b"}]}"#),
]);
assert_eq!(h.add_server("Npu", &flm, None, None).unwrap(), "qwen3.8-mtp-uncensored:27b", "the loaded chat model, not an embedding one");
assert_eq!(h.add_server("Named", &flm, Some("gemma3:1b"), None).unwrap(), "gemma3:1b", "a named model still wins");
let idle = serve_routes(&[("/v1/models", CATALOGUE), ("/api/ps", r#"{"models":[]}"#)]);
let no_ps = serve_routes(&[("/v1/models", CATALOGUE)]);
for (name, url) in [("Idle", &idle), ("Studio", &no_ps)] {
let e = h.add_server(name, url, None, None).unwrap_err();
assert_eq!(e.code, ErrorCode::BadInput, "{name}: {e:?}");
assert_eq!(models_to_choose(&e), ["deepseek-r1:8b", "gemma3:1b", "qwen3.8-mtp-uncensored:27b"], "{name}");
assert!(e.message.contains("lists 3 chat models"), "{e:?}");
assert!(e.remediation.as_deref().is_some_and(|r| r.contains("deepseek-r1:8b, gemma3:1b")), "{e:?}");
}
assert_eq!(h.add_server("Studio", &no_ps, Some("gemma3:1b"), None).unwrap(), "gemma3:1b", "the chosen one is added");
// One chat model beside an embedding one: nothing to choose.
let one = serve_routes(&[("/v1/models", r#"{"object":"list","data":[{"id":"nomic-embed-text"},{"id":"qwen3:8b"}]}"#)]);
assert_eq!(h.add_server("One", &one, None, None).unwrap(), "qwen3:8b");
assert!(models_to_choose(&ToolError::invalid("other")).is_empty(), "other errors offer nothing");
let names: Vec<_> = h.backends().into_iter().map(|b| b.name).collect();
assert!(!names.iter().any(|n| n == "Idle"), "nothing added while a choice is owed: {names:?}");
}
/// #54, #98: a server on another computer is added by its address, with a model that can chat
/// when none is named; one that doesn't answer with its models isn't added.
#[test]
fn a_server_is_added_by_its_address() {
let (h, _d) = hub();
let url = serve("200 OK", r#"{"object":"list","data":[{"id":"nomic-embed-text"},{"id":"qwen3:8b"}]}"#);
assert_eq!(h.add_server(" Studio box ", &format!("{url}/"), None, Some(" sk-test ".into())).unwrap(), "qwen3:8b");
let b = h.backends().into_iter().find(|b| b.name == "Studio box").unwrap();
assert_eq!((b.kind, b.model.as_str(), b.base_url.as_deref()), (BackendKind::OpenaiCompatible, "qwen3:8b", Some(url.as_str())));
assert_eq!(h.add_server("Named", &url, Some("llama3"), None).unwrap(), "llama3", "a named model is kept");
assert!(h.add_server("Web page", &serve("404 Not Found", "nope"), None, None).is_err(), "not a model server");
assert!(h.add_server("Empty", &serve("200 OK", r#"{"object":"list","data":[]}"#), None, None).is_err(), "no models to use");
let embeddings = serve("200 OK", r#"{"object":"list","data":[{"id":"nomic-embed-text"}]}"#);
let error = h.add_server("Embeddings", &embeddings, None, None).unwrap_err();
assert_eq!(error.code, ErrorCode::NotFound);
assert!(error.message.contains("model that can chat"));
assert!(h.add_server("No scheme", "192.168.1.20:11434/v1", None, None).is_err());
assert!(h.add_server(" ", &url, None, None).is_err());
assert_eq!(h.backends().len(), 2, "only the ones that answered");
}
/// #1429: a server that takes the request and doesn't list its models in time (still starting, or
/// loading a model) is told to wait, not to check its address, and the message isn't repeated.
#[test]
fn a_server_that_was_reached_but_slow_is_told_to_wait() {
let (h, _d) = hub();
let l = TcpListener::bind("127.0.0.1:0").unwrap();
let url = format!("http://{}/v1", l.local_addr().unwrap());
// Accepts and holds the connection without answering, for longer than the test waits.
std::thread::spawn(move || {
let held: Vec<_> = l.incoming().take(1).flatten().collect();
std::thread::sleep(Duration::from_secs(10));
drop(held);
});
let e = h.add_server_within("Slow box", &url, None, None, Duration::from_secs(2), &Default::default()).unwrap_err();
assert_eq!(e.code, ErrorCode::TimedOut, "{e:?}");
let advice = e.remediation.clone().unwrap_or_default();
assert!(advice.contains("was reached") && advice.contains("2 seconds") && advice.contains("loading a model"), "{advice}");
assert!(!advice.contains("Check the address") && !advice.contains(&e.message), "{advice}");
assert!(h.backends().is_empty(), "nothing added");
// Nothing listening (port zero refuses): still told to check the address, without the message
// again in brackets.
let e = h.add_server("Nobody", "http://127.0.0.1:0/v1", None, None).unwrap_err();
let advice = e.remediation.clone().unwrap_or_default();
assert!(advice.contains("Check the address") && !advice.contains(&e.message), "{advice}");
}
/// #1429: adding a server that was reached but doesn't answer can be stopped while it's asked,
/// instead of waiting out the minute, and nothing is added.
#[test]
fn adding_a_slow_server_can_be_stopped() {
let (h, _d) = hub();
let l = TcpListener::bind("127.0.0.1:0").unwrap();
let url = format!("http://{}/v1", l.local_addr().unwrap());
std::thread::spawn(move || {
let held: Vec<_> = l.incoming().take(1).flatten().collect();
std::thread::sleep(Duration::from_secs(10));
drop(held);
});
let cancel = craft_anything_engine::CancelToken::default();
let later = cancel.clone();
std::thread::spawn(move || {
std::thread::sleep(Duration::from_millis(300));
later.cancel();
});
let started = std::time::Instant::now();
let e = h.add_server_until("Slow box", &url, None, None, &cancel).unwrap_err();
assert_eq!(e.code, ErrorCode::Cancelled, "{e:?}");
assert!(started.elapsed() < Duration::from_secs(5), "stopped at once, not after the wait: {:?}", started.elapsed());
assert!(e.remediation.is_none(), "a stop isn't a bad address: {e:?}");
assert!(h.backends().is_empty(), "nothing added");
}
#[test]
fn a_backend_context_budget_reaches_the_model() {
let (hub, dir) = hub();
let table: toml::Table = toml::from_str(
"[[backends]]\nname = \"Local\"\nkind = \"openai_compatible\"\nmodel = \"qwen3:8b\"\nbase_url = \"http://127.0.0.1:9/v1\"\ncontext_tokens = 8000\n",
)
.unwrap();
let entry = AiConfig::from_table(&table).unwrap().backends.remove(0);
hub.add_backend_with(entry, None).unwrap();
let slot = hub.build_model(&hub.entry("Local").unwrap()).unwrap();
assert_eq!(slot.context_tokens, 8000);
let saved = AiConfig::from_settings(&Settings::load_from(&dir.path().join("config.toml")));
assert_eq!(saved.get("Local").unwrap().options.context_tokens, Some(8000));
}
/// #374: a remote OpenAI-compatible service was treated as a local model: a 24K-token context
/// and a cost of nothing. Only a server on this computer or its network is local.
#[test]
fn only_an_openai_compatible_server_nearby_is_local() {
let (h, _d) = hub();
for (name, base, local) in [
("Ollama", "http://127.0.0.1:11434/v1", true),
("LM Studio box", "http://192.168.1.20:1234/v1", true),
("Docker Ollama", "http://ollama:11434/v1", true),
("IPv6 link-local", "http://[fe80::1]:11434/v1", true),
("Z.ai", "https://api.z.ai/api/coding/paas/v4", false),
("Public vLLM", "http://8.8.8.8:8000/v1", false),
] {
add(&h, name, BackendKind::OpenaiCompatible, base, None);
let slot = h.build_model(&h.entry(name).unwrap()).unwrap();
assert_eq!(slot.local, local, "{name} at {base}");
}
h.add_backend(
BackendConfig { name: "Default".into(), kind: BackendKind::OpenaiCompatible, model: String::new(), base_url: None, ssh_host: None, enabled: true },
None,
)
.unwrap();
assert!(h.build_model(&h.entry("Default").unwrap()).unwrap().local, "no address means Ollama on this computer");
add(&h, "OpenAI", BackendKind::Openai, "http://127.0.0.1:9/v1", Some("sk-test"));
assert!(!h.build_model(&h.entry("OpenAI").unwrap()).unwrap().local, "only OpenAI-compatible servers can be local");
}
fn add(h: &AiHub, name: &str, kind: BackendKind, base: &str, key: Option<&str>) {
h.add_backend(
BackendConfig { name: name.into(), kind, model: String::new(), base_url: Some(base.into()), ssh_host: None, enabled: true },
key.map(str::to_string),
)
.unwrap();
}
fn run(c: &mut Box<dyn Conversation>, text: &str) -> Vec<AgentEvent> {
c.send(text);
let deadline = Instant::now() + Duration::from_secs(20);
let mut all = Vec::new();
while Instant::now() < deadline {
for e in c.poll() {
let done = matches!(e, AgentEvent::TurnDone { .. });
all.push(e);
if done {
return all;
}
}
std::thread::sleep(Duration::from_millis(5));
}
panic!("timed out: {all:?}");
}
#[test]
fn anthropic_streaming_tool_loop_through_the_engine() {
let turn1 = sse(&[
json!({"type":"message_start","message":{"model":"claude-opus-5-5","usage":{"input_tokens":100,"output_tokens":1}}}),
json!({"type":"ping"}),
json!({"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":""}}),
json!({"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"plan"}}),
json!({"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"SIG"}}),
json!({"type":"content_block_stop","index":0}),
json!({"type":"content_block_start","index":1,"content_block":{"type":"text","text":""}}),
json!({"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"Checking"}}),
json!({"type":"content_block_stop","index":1}),
json!({"type":"content_block_start","index":2,"content_block":{"type":"tool_use","id":"tu_1","name":"core_session_show","input":{}}}),
json!({"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"{"}}),
json!({"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"}"}}),
json!({"type":"content_block_stop","index":2}),
json!({"type":"content_block_start","index":3,"content_block":{"type":"tool_use","id":"tu_2","name":"core_targets_list","input":{}}}),
json!({"type":"content_block_stop","index":3}),
json!({"type":"message_delta","delta":{"stop_reason":"tool_use"},"usage":{"output_tokens":40}}),
json!({"type":"message_stop"}),
]);
let turn2 = sse(&[
json!({"type":"message_start","message":{"model":"claude-opus-5-5","usage":{"input_tokens":200,"cache_read_input_tokens":90}}}),
json!({"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}),
json!({"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" Nothing is open."}}),
json!({"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":10}}),
json!({"type":"message_stop"}),
]);
let (url, seen) = mock(vec![(200, "text/event-stream", turn1), (200, "text/event-stream", turn2)]);
let (h, _d) = hub();
add(&h, "Claude", BackendKind::Anthropic, &url, Some("sk-test-123"));
let mut c = h.start(None, true).unwrap();
let ev = run(&mut c, "How does the session look?");
let text: String = ev.iter().filter_map(|e| if let AgentEvent::TextDelta { text } = e { Some(text.as_str()) } else { None }).collect();
assert_eq!(text, "Checking Nothing is open.");
assert!(ev.iter().any(|e| matches!(e, AgentEvent::ToolFinished { tool, ok: true, .. } if tool == "core_session_show")));
assert!(ev.iter().any(|e| matches!(e, AgentEvent::ToolFinished { tool, ok: true, .. } if tool == "core_targets_list")));
let AgentEvent::TurnDone { backend, input_tokens, output_tokens, cost_usd, .. } = ev.last().unwrap().clone() else { panic!() };
// The 90 cached tokens count as input, as agent apps report them.
assert_eq!((backend.as_str(), input_tokens, output_tokens), ("Claude", 390, 50));
assert!(cost_usd.unwrap() > 0.0);
let seen = seen.lock().unwrap();
assert_eq!(seen.len(), 2);
let r1 = &seen[0];
assert_eq!(r1.path, "/v1/messages");
assert_eq!(r1.header("x-api-key"), Some("sk-test-123"));
assert_eq!(r1.header("anthropic-version"), Some("2023-06-01"));
assert_eq!(r1.header("anthropic-beta"), Some("server-side-fallback-2026-07-01"));
assert_eq!(r1.header("user-agent"), Some("CraftAnything-dev"));
let b = &r1.body;
assert_eq!(b["model"], "claude-opus-5-5");
assert_eq!(b["stream"], true);
assert_eq!(b["max_tokens"], 64000);
assert_eq!(b["thinking"]["type"], "adaptive");
assert_eq!(b["output_config"]["effort"], "high");
assert_eq!(b["fallbacks"], "default");
assert_eq!(b["tool_choice"]["type"], "auto");
assert_eq!(b["system"][0]["cache_control"]["type"], "ephemeral");
assert!(b["system"][0]["text"].as_str().unwrap().contains("Investigation so far"));
let tools = b["tools"].as_array().unwrap();
let names: Vec<&str> = tools.iter().map(|t| t["name"].as_str().unwrap()).collect();
let mut sorted = names.clone();
sorted.sort();
assert_eq!(names, sorted, "tool order is deterministic");
assert!(names.contains(&"core_session_show") && names.contains(&"enable_toolset") && names.contains(&"core_handoff_write"));
assert!(!names.contains(&"native_cursor_show"), "non-Core tools wait for enable_toolset");
assert_eq!(tools.last().unwrap()["cache_control"]["type"], "ephemeral");
assert!(tools.iter().all(|t| t["input_schema"]["type"] == "object"));
// Turn 2 replays thinking (with signature) and returns BOTH results in ONE user message.
let msgs = seen[1].body["messages"].as_array().unwrap();
assert_eq!(msgs.len(), 3);
assert_eq!(msgs[1]["content"][0], json!({"type":"thinking","thinking":"plan","signature":"SIG"}));
let results = msgs[2]["content"].as_array().unwrap();
assert_eq!(msgs[2]["role"], "user");
assert_eq!(results.len(), 2);
assert_eq!(results[0]["tool_use_id"], "tu_1");
assert_eq!(results[1]["tool_use_id"], "tu_2");
assert_eq!(results[0]["is_error"], false);
assert!(results[0]["content"].as_str().unwrap().contains("deep"), "test sessions are in Deep mode");
// the key is never stored in settings
let saved = std::fs::read_to_string(_d.path().join("config.toml")).unwrap();
assert!(!saved.contains("sk-test-123"));
}
#[test]
fn anthropic_refusal_and_http_errors() {
let refusal = sse(&[
json!({"type":"message_start","message":{"model":"claude-opus-5-5","usage":{"input_tokens":5}}}),
json!({"type":"message_delta","delta":{"stop_reason":"refusal"},"usage":{"output_tokens":0}}),
json!({"type":"message_stop"}),
]);
let (url, _seen) = mock(vec![
(200, "text/event-stream", refusal),
(401, "application/json", r#"{"type":"error","error":{"type":"authentication_error","message":"invalid x-api-key"}}"#.into()),
]);
let (h, _d) = hub();
add(&h, "Claude", BackendKind::Anthropic, &url, Some("k"));
let mut c = h.start(Some("Claude"), false).unwrap();
let ev = run(&mut c, "q");
assert!(ev.iter().any(|e| matches!(e, AgentEvent::Error { message, .. } if message.contains("declined"))));
let ev = run(&mut c, "q2");
assert!(
ev.iter().any(|e| matches!(e, AgentEvent::Error { message, remediation: Some(r), .. } if message.contains("invalid x-api-key") && r.contains("key"))),
"{ev:?}"
);
}
/// A service, or a proxy in front of it, can repeat the key or words from the request in its
/// error: in a JSON error's message, type or code, in a plain-text body, or in a stream's error
/// event. None of it reaches the failure log, in the error, the service's code or the advice. The
/// marker is written like an error code (lower-case letters and an underscore), so only a list of
/// known codes keeps it out; a known code is still kept (R5.5).
#[test]
fn a_failure_log_leaves_out_what_the_service_repeats_of_the_key_and_the_request() {
let key = "sk-test-reflected-key";
let marker = "quokka_umbrella";
let reflected = format!("key {key} may not ask about the {marker}");
let stream_error = |kind: &str| {
sse(&[
json!({"type":"message_start","message":{"model":"claude-opus-5-5","usage":{"input_tokens":5}}}),
json!({"type":"error","error":{"type":kind,"message":reflected}}),
])
};
let (url, _seen) = mock(vec![
(401, "application/json", json!({"type":"error","error":{"type":marker,"message":reflected}}).to_string()),
(400, "application/json", json!({"error":{"code":marker,"message":reflected}}).to_string()),
(400, "text/plain", format!("Rejected by the proxy: {reflected}")),
(200, "text/event-stream", stream_error(marker)),
(200, "text/event-stream", stream_error("invalid_request_error")),
]);
let (h, _d) = hub();
add(&h, "Claude", BackendKind::Anthropic, &url, Some(key));
let mut c = h.start(Some("Claude"), false).unwrap();
let unknown = "The service's error code: not a known one";
let expected: [&[&str]; 5] = [
&["Error: NoPermission", "HTTP status: 401", unknown, "Diagnosis: The API key was rejected"],
&["Error: BadInput", "HTTP status: 400", unknown],
&["Error: BadInput", "HTTP status: 400"],
&["Error: ProgramFailed", unknown, "Diagnosis: Try again"],
&["Error: ProgramFailed", "The service's error code: invalid_request_error"],
];
for (n, parts) in expected.iter().enumerate() {
let ev = run(&mut c, &format!("a question about the {marker}, take {n}"));
let log = ev.iter().find_map(|e| if let AgentEvent::Error { log, .. } = e { log.clone() } else { None });
let text = std::fs::read_to_string(log.unwrap_or_else(|| panic!("no log for take {n}: {ev:?}"))).unwrap();
for hidden in [key, marker, "may not ask"] {
assert!(!text.contains(hidden), "take {n}, {hidden}: {text}");
}
for part in *parts {
assert!(text.contains(part), "take {n}, {part}: {text}");
}
}
}
#[test]
fn openai_streaming_tool_loop() {
let t1 = chat_sse(&[
json!({"model":"gpt-x","choices":[{"delta":{"role":"assistant","tool_calls":[{"index":0,"id":"call_a","type":"function","function":{"name":"core_session_show","arguments":""}}]}}]}),
json!({"choices":[{"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{}"}}]}}]}),
json!({"choices":[{"delta":{},"finish_reason":"tool_calls"}]}),
json!({"choices":[],"usage":{"prompt_tokens":20,"completion_tokens":3}}),
]);
let t2 = chat_sse(&[json!({"choices":[{"delta":{"content":"Nothing open."},"finish_reason":"stop"}]})]);
let (url, seen) = mock(vec![(200, "text/event-stream", t1), (200, "text/event-stream", t2)]);
let (h, _d) = hub();
add(&h, "GPT", BackendKind::Openai, &url, Some("sk-o"));
let mut c = h.start(Some("GPT"), true).unwrap();
let ev = run(&mut c, "status?");
assert!(ev.iter().any(|e| matches!(e, AgentEvent::TextDelta { text } if text == "Nothing open.")));
let seen = seen.lock().unwrap();
assert_eq!(seen[0].path, "/chat/completions");
assert_eq!(seen[0].header("authorization"), Some("Bearer sk-o"));
assert_eq!(seen[0].body["stream_options"]["include_usage"], true);
assert_eq!(seen[0].body["tool_choice"], "auto");
let msgs = seen[1].body["messages"].as_array().unwrap();
assert_eq!(msgs[2]["tool_calls"][0]["id"], "call_a");
assert_eq!(msgs[3]["role"], "tool");
assert_eq!(msgs[3]["tool_call_id"], "call_a");
}
#[test]
fn native_tool_stop_reasons_are_respected_before_execution() {
for (finish, run_tool) in [("length", false), ("content_filter", false), ("stop", true), ("tool_calls", true)] {
let first = chat_sse(&[
json!({"choices":[{"delta":{"tool_calls":[{
"index":0,"id":"call_a","type":"function","function":{"name":"core_session_show","arguments":"{}"}
}]}}]}),
json!({"choices":[{"delta":{},"finish_reason":finish}]}),
]);
let mut responses = vec![(200, "text/event-stream", first)];
if finish != "content_filter" {
let last = chat_sse(&[json!({"choices":[{"delta":{"content":"Done."},"finish_reason":"stop"}]})]);
responses.push((200, "text/event-stream", last));
}
let (url, seen) = mock(responses);
let (h, _dir) = hub();
add(&h, "GPT", BackendKind::Openai, &url, Some("sk-test"));
let mut conversation = h.start(Some("GPT"), false).unwrap();
let events = run(&mut conversation, "status?");
let started = events.iter().filter(|event| matches!(event, AgentEvent::ToolStarted { .. })).count();
let completed = events.iter().filter(|event| matches!(event, AgentEvent::ToolFinished { tool, ok: true, .. } if tool == "core_session_show")).count();
assert_eq!(started, usize::from(run_tool), "finish_reason={finish}: {events:?}");
assert_eq!(completed, usize::from(run_tool), "finish_reason={finish}: {events:?}");
let seen = seen.lock().unwrap();
if finish == "content_filter" {
assert_eq!(seen.len(), 1, "a refused turn must not request a continuation");
assert!(events.iter().any(|event| matches!(event, AgentEvent::Error { message, .. } if message.contains("declined"))));
} else {
assert_eq!(seen.len(), 2);
let messages = seen[1].body["messages"].as_array().unwrap();
let result = messages.last().unwrap();
assert_eq!(result["role"], "tool");
assert_eq!(result["tool_call_id"], "call_a");
assert_eq!(result["content"].as_str().unwrap().starts_with("Not run: the reply hit the output limit"), finish == "length");
assert!(events.iter().any(|event| matches!(event, AgentEvent::TextDelta { text } if text == "Done.")));
}
}
}
fn assert_unexecuted_native_tool_calls_are_paired(finish: &str) {
let first = chat_sse(&[
json!({"choices":[{"delta":{"content":"Stopped before executing.","tool_calls":[
{"index":0,"id":"call_a","type":"function","function":{"name":"core_session_show","arguments":"{}"}},
{"index":1,"id":"call_b","type":"function","function":{"name":"core_session_show","arguments":"{}"}}
]}}]}),
json!({"choices":[{"delta":{},"finish_reason":finish}]}),
]);
let last = chat_sse(&[json!({"choices":[{"delta":{"content":"Done."},"finish_reason":"stop"}]})]);
let (url, seen) = mock(vec![(200, "text/event-stream", first), (200, "text/event-stream", last)]);
let (h, _dir) = hub();
add(&h, "GPT", BackendKind::Openai, &url, Some("sk-test"));
let mut conversation = h.start(Some("GPT"), false).unwrap();
let mut events = run(&mut conversation, "status?");
if finish != "pause_turn" {
assert_eq!(seen.lock().unwrap().len(), 1, "{finish} must wait for the next user turn");
events.extend(run(&mut conversation, "Continue when ready."));
}
assert!(
!events.iter().any(|event| matches!(event, AgentEvent::ToolStarted { .. } | AgentEvent::ToolFinished { .. } | AgentEvent::ConsentNeeded { .. })),
"stopped calls must not run or ask for consent: {events:?}"
);
assert!(!events.iter().any(|event| matches!(event, AgentEvent::Error { .. })), "{events:?}");
assert!(events.iter().any(|event| matches!(event, AgentEvent::TextDelta { text } if text == "Done.")), "{events:?}");
let seen = seen.lock().unwrap();
assert_eq!(seen.len(), 2, "{finish}: one stopped response and one continuation");
let messages = seen[1].body["messages"].as_array().unwrap();
let call_at = messages.iter().position(|message| message["tool_calls"].is_array()).unwrap();
assert_eq!(messages[call_at]["tool_calls"].as_array().unwrap().len(), 2);
assert_eq!(messages[call_at]["content"], "Stopped before executing.");
for (offset, id) in ["call_a", "call_b"].into_iter().enumerate() {
let result = messages.get(call_at + offset + 1);
assert_eq!(result.and_then(|message| message["role"].as_str()), Some("tool"), "{finish}: each retained call needs a result: {messages:?}");
let result = result.unwrap();
assert_eq!(result["tool_call_id"], id);
assert!(result["content"].as_str().unwrap().starts_with("Not run:"), "{result:?}");
}
if finish == "pause_turn" {
assert_eq!(messages.last().unwrap()["tool_call_id"], "call_b", "a pause continues without another user request");
} else {
assert_eq!(messages.last().unwrap()["role"], "user");
assert_eq!(messages.last().unwrap()["content"], "Continue when ready.");
}
}
#[test]
fn paused_native_tool_calls_keep_history_paired() {
assert_unexecuted_native_tool_calls_are_paired("pause_turn");
}
#[test]
fn sequence_stopped_native_tool_calls_keep_history_paired() {
assert_unexecuted_native_tool_calls_are_paired("stop_sequence");
}
#[test]
fn provider_stopped_native_tool_calls_keep_history_paired() {
assert_unexecuted_native_tool_calls_are_paired("provider_stop");
}
#[test]
fn local_server_without_tools_falls_back_to_react() {
let t1 = chat_sse(&[json!({"choices":[{"delta":{"content":"Thought: check\nAction: core_session_show\nAction Input: {}"},"finish_reason":"stop"}]})]);
let t2 = chat_sse(&[json!({"choices":[{"delta":{"content":"Final Answer: nothing is open"},"finish_reason":"stop"}]})]);
let (url, seen) = mock(vec![
(400, "application/json", r#"{"error":{"message":"registry.ollama.ai/library/tiny does not support tools"}}"#.into()),
(200, "text/event-stream", t1),
(200, "text/event-stream", t2),
]);
let (h, _d) = hub();
add(&h, "Tiny", BackendKind::OpenaiCompatible, &format!("{url}/v1"), None);
let mut c = h.start(Some("Tiny"), true).unwrap();
let ev = run(&mut c, "status?");
assert!(ev.iter().any(|e| matches!(e, AgentEvent::ToolFinished { tool, ok: true, .. } if tool == "core_session_show")), "{ev:?}");
assert!(matches!(ev.last(), Some(AgentEvent::TurnDone { cost_usd: Some(c), .. }) if *c == 0.0));
let seen = seen.lock().unwrap();
assert!(seen[0].body.get("tools").is_some());
assert!(seen[0].header("authorization").is_none());
assert!(seen[1].body.get("tools").is_none());
let last = seen[2].body["messages"].as_array().unwrap().last().unwrap().clone();
assert!(last["content"].as_str().unwrap().starts_with("Observation:"));
}
/// #498: return a field-specific error to the model, then accept its corrected call.
#[test]
fn a_model_can_correct_an_array_sent_to_a_string_parameter() {
let call = |id: &str, input: Value| {
chat_sse(&[json!({"choices":[{
"delta":{"tool_calls":[{"index":0,"id":id,"type":"function","function":{
"name":"core_unknown_add","arguments":input.to_string()
}}]},"finish_reason":"tool_calls"
}]})])
};
let (url, seen) = mock(vec![
(200, "text/event-stream", call("wrong", json!({"question": ["Where is the loop?"]}))),
(200, "text/event-stream", call("corrected", json!({"question": "Where is the loop?"}))),
(200, "text/event-stream", chat_sse(&[json!({"choices":[{"delta":{"content":"Recorded."},"finish_reason":"stop"}]})])),
]);
let (h, _d) = hub();
add(&h, "Local", BackendKind::OpenaiCompatible, &url, None);
let mut c = h.start(Some("Local"), true).unwrap();
let events = run(&mut c, "Record what remains unknown.");
let completed: Vec<_> = events
.iter()
.filter_map(|event| match event {
AgentEvent::ToolFinished { tool, ok, .. } if tool == "core_unknown_add" => Some(*ok),
_ => None,
})
.collect();
assert_eq!(completed, vec![false, true]);
assert_eq!(h.engine.session().unknowns().count(), 1, "only the corrected call creates a record");
let seen = seen.lock().unwrap();
assert_eq!(seen.len(), 3);
let result = seen[1].body["messages"].as_array().unwrap().last().unwrap();
assert_eq!(result["role"], "tool");
assert_eq!(result["tool_call_id"], "wrong");
let error: Value = serde_json::from_str(result["content"].as_str().unwrap()).unwrap();
assert_eq!(error["error"]["code"], "bad_input");
assert_eq!(error["error"]["details"]["parameter_path"], "/question");
assert!(error["error"]["message"].as_str().unwrap().contains("expected a string"));
assert!(error["error"]["remediation"].as_str().unwrap().contains("call the tool again"));
}
#[test]
fn json_text_tool_calls_execute_through_both_protocols() {
for react in [false, true] {
for call in [
r#"{"name":"core_session_show","arguments":{}}"#,
"```json\n{\"name\":\"core_session_show\",\"arguments\":{}}\n````",
"````json\n{\"name\":\"core_session_show\",\"arguments\":{}}\n````",
] {
let t1 = chat_sse(&[json!({"choices":[{"delta":{"content":call},"finish_reason":"stop"}]})]);
let t2 = chat_sse(&[json!({"choices":[{"delta":{"content":"Nothing open."},"finish_reason":"stop"}]})]);
let (url, seen) = mock(vec![(200, "text/event-stream", t1), (200, "text/event-stream", t2)]);
let (hub, _dir) = hub();
let config = BackendConfig {
name: "Local".into(),
kind: BackendKind::OpenaiCompatible,
model: "scripted".into(),
base_url: Some(url),
ssh_host: None,
enabled: true,
};
hub.add_backend_with(BackendEntry { config, options: BackendOptions { react_fallback: Some(react), ..Default::default() } }, None).unwrap();
let mut conversation = hub.start(Some("Local"), true).unwrap();
let events = run(&mut conversation, "status?");
assert_eq!(
events.iter().filter(|e| matches!(e, AgentEvent::ToolFinished { tool, ok: true, .. } if tool == "core_session_show")).count(),
1,
"react={react}, call={call}: {events:?}"
);
assert!(events.iter().any(|e| matches!(e, AgentEvent::TextDelta { text } if text == "Nothing open.")), "{events:?}");
let seen = seen.lock().unwrap();
assert_eq!(seen.len(), 2);
let messages = seen[1].body["messages"].as_array().unwrap();
let result = messages.last().unwrap();
if react {
assert_eq!(result["role"], "user");
assert!(result["content"].as_str().unwrap().starts_with("Observation:"));
} else {
assert_eq!(result["role"], "tool");
assert_eq!(result["tool_call_id"], messages[2]["tool_calls"][0]["id"]);
assert_eq!(messages[2]["tool_calls"][0]["function"]["name"], "core_session_show");
}
}
}
}
#[test]
fn react_action_errors_still_reach_the_tool_loop() {
let replies = ["Action: missing_tool\nAction Input: {}", "Action: core_session_show\nAction Input: {}", "Final Answer: Nothing open."]
.map(|text| (200, "text/event-stream", chat_sse(&[json!({"choices":[{"delta":{"content":text},"finish_reason":"stop"}]})])));
let (url, seen) = mock(replies.into());
let (hub, _dir) = hub();
let config = BackendConfig {
name: "Local".into(),
kind: BackendKind::OpenaiCompatible,
model: "scripted".into(),
base_url: Some(url),
ssh_host: None,
enabled: true,
};
hub.add_backend_with(BackendEntry { config, options: BackendOptions { react_fallback: Some(true), ..Default::default() } }, None).unwrap();
let mut conversation = hub.start(Some("Local"), true).unwrap();
let events = run(&mut conversation, "status?");
assert!(events.iter().any(|e| matches!(e, AgentEvent::ToolFinished { tool, ok: false, .. } if tool == "missing_tool")), "{events:?}");
assert!(events.iter().any(|e| matches!(e, AgentEvent::ToolFinished { tool, ok: true, .. } if tool == "core_session_show")), "{events:?}");
let seen = seen.lock().unwrap();
assert_eq!(seen.len(), 3);
let correction = seen[1].body["messages"].as_array().unwrap().last().unwrap()["content"].as_str().unwrap();
assert!(correction.starts_with("Observation (error):") && correction.contains("No tool named missing_tool"), "{correction}");
}
#[test]
fn fenced_legacy_react_calls_still_execute() {
for call in [
"```text\nAction: core_session_show\nAction Input: {}\n```",
"```text\nAction: core_session_show\nAction Input: {}\n````",
"````text\nAction: core_session_show\nAction Input: {}\n````",
"````text\nAction: core_session_show\nAction Input: ```json\n{}\n```\n````",
] {
let replies = [call, "Final Answer: Nothing open."]
.map(|text| (200, "text/event-stream", chat_sse(&[json!({"choices":[{"delta":{"content":text},"finish_reason":"stop"}]})])));
let (url, seen) = mock(replies.into());
let (hub, _dir) = hub();
let config = BackendConfig {
name: "Local".into(),
kind: BackendKind::OpenaiCompatible,
model: "scripted".into(),
base_url: Some(url),
ssh_host: None,
enabled: true,
};
hub.add_backend_with(BackendEntry { config, options: BackendOptions { react_fallback: Some(true), ..Default::default() } }, None).unwrap();
let mut conversation = hub.start(Some("Local"), true).unwrap();
let events = run(&mut conversation, "status?");
assert_eq!(
events.iter().filter(|e| matches!(e, AgentEvent::ToolFinished { tool, ok: true, .. } if tool == "core_session_show")).count(),
1,
"the fenced legacy call must execute exactly once: {events:?}"
);
assert!(events.iter().any(|e| matches!(e, AgentEvent::TextDelta { text } if text == "Final Answer: Nothing open.")), "{events:?}");
let seen = seen.lock().unwrap();
assert_eq!(seen.len(), 2);
let observation = seen[1].body["messages"].as_array().unwrap().last().unwrap()["content"].as_str().unwrap();
assert!(observation.starts_with("Observation:"), "{observation}");
}
}
#[test]
fn a_qwen_markup_call_to_a_missing_tool_is_answered_and_the_build_goes_on() {
// A live build stopped here: the server handed back a call to a tool it wasn't offered as
// text, and the loop took the text for the final answer.
let missing = "<tool_call>\n<function=web_search>\n<parameter=query>\neframe screenshot example\n</parameter>\n</function>\n</tool_call>";
let real = "<tool_call>\n<function=core_session_show>\n</function>\n</tool_call>";
let replies = [missing, real, "Nothing open."]
.map(|text| (200, "text/event-stream", chat_sse(&[json!({"choices":[{"delta":{"content":text},"finish_reason":"stop"}]})])));
let (url, seen) = mock(replies.into());
let (hub, _dir) = hub();
let config = BackendConfig {
name: "Local".into(),
kind: BackendKind::OpenaiCompatible,
model: "scripted".into(),
base_url: Some(url),
ssh_host: None,
enabled: true,
};
hub.add_backend_with(BackendEntry { config, options: BackendOptions::default() }, None).unwrap();
let mut conversation = hub.start(Some("Local"), true).unwrap();
let events = run(&mut conversation, "status?");
assert_eq!(events.iter().filter(|e| matches!(e, AgentEvent::ToolFinished { tool, ok: true, .. } if tool == "core_session_show")).count(), 1, "{events:?}");
assert!(!events.iter().any(|e| matches!(e, AgentEvent::ToolStarted { tool, .. } if tool == "web_search")), "{events:?}");
let seen = seen.lock().unwrap();
assert_eq!(seen.len(), 3, "the turn must go on after the missing tool");
let told = seen[1].body["messages"].as_array().unwrap().last().unwrap();
assert_eq!(told["role"], "user");
let told = told["content"].as_str().unwrap();
assert!(told.contains("`web_search` isn't a tool you can use here") && told.contains("core_session_show"), "{told}");
// The call that ran is paired with its result in the next request.
let last = seen[2].body["messages"].as_array().unwrap().last().unwrap();
assert_eq!(last["role"], "tool", "{last}");
}
#[test]
fn json_text_recovery_preserves_non_actions_and_stop_reasons() {
use crate::model::{Effort, Msg, Stop, StopReason, ToolDef, TurnRequest};
let tools = [ToolDef { name: "look".into(), description: "Look at a file".into(), schema: json!({"type":"object"}) }];
let messages = [Msg::user_text("check")];
let request = TurnRequest { system: "S", messages: &messages, tools: &tools, effort: Effort::Low, max_tokens: 100 };
for react in [false, true] {
for (text, finish, expected) in [
(r#"{"name":"unknown","arguments":{}}"#, "stop", StopReason::EndTurn),
(r#"{"name":"look","arguments":{}}"#, "length", StopReason::MaxTokens),
(r#"{"name":"look","arguments":{}}"#, "content_filter", StopReason::Refusal),
(r#"{"name":"look","arguments":{}"#, "stop", StopReason::EndTurn),
(r#"{"answer":42}"#, "stop", StopReason::EndTurn),
(r#"For example: {"name":"look","arguments":{}}"#, "stop", StopReason::EndTurn),
("```text\nAction: look\nAction Input: {}\n```not a close```", "stop", StopReason::EndTurn),
("```json\n{\"name\":\"look\",\"arguments\":{}}```", "stop", StopReason::EndTurn),
("```text\nAction: look\nAction Input: {}```", "stop", StopReason::EndTurn),
("````text\nAction: look\nAction Input: {}\n```", "stop", StopReason::EndTurn),
("```text\nAction: look\nAction Input: {}\n```\nThis is only an example.\n```", "stop", StopReason::EndTurn),
] {
let reply = chat_sse(&[json!({"choices":[{"delta":{"content":text},"finish_reason":finish}]})]);
let (url, _seen) = mock(vec![(200, "text/event-stream", reply)]);
let mut model = openai::OpenAi::compatible("Local", &url, None, "scripted");
model.react = react;
let result = model.turn(&request, &mut |_| {}, &Stop::default()).unwrap();
assert!(result.message.tool_uses().is_empty(), "{result:?}");
assert_eq!(result.stop, expected, "{text}");
}
}
let call = r#"{"name":"look","arguments":{}}"#;
for (automatic, offered) in [(false, tools.as_slice()), (true, &[])] {
let reply = chat_sse(&[json!({"choices":[{"delta":{"content":call},"finish_reason":"stop"}]})]);
let (url, _seen) = mock(vec![(200, "text/event-stream", reply)]);
let mut model = openai::OpenAi::compatible("Local", &url, None, "scripted");
model.auto_react = automatic;
let request = TurnRequest { system: "S", messages: &messages, tools: offered, effort: Effort::Low, max_tokens: 100 };
assert!(model.turn(&request, &mut |_| {}, &Stop::default()).unwrap().message.tool_uses().is_empty());
}
}
#[test]
fn native_tool_calls_are_not_duplicated_by_json_text() {
use crate::model::{Effort, Msg, Stop, StopReason, ToolDef, TurnRequest};
let tools = [ToolDef { name: "look".into(), description: "Look at a file".into(), schema: json!({"type":"object"}) }];
let messages = [Msg::user_text("check")];
let request = TurnRequest { system: "S", messages: &messages, tools: &tools, effort: Effort::Low, max_tokens: 100 };
for react in [false, true] {
let reply = chat_sse(&[json!({"choices":[{"delta":{
"content":r#"{"name":"look","arguments":{}}"#,
"tool_calls":[{"index":0,"id":"native_call","type":"function","function":{"name":"look","arguments":"{}"}}]
},"finish_reason":"tool_calls"}]})]);
let (url, _seen) = mock(vec![(200, "text/event-stream", reply)]);
let mut model = openai::OpenAi::compatible("Local", &url, None, "scripted");
model.react = react;
let result = model.turn(&request, &mut |_| {}, &Stop::default()).unwrap();
assert_eq!(result.message.tool_uses(), vec![("native_call".into(), "look".into(), json!({}))]);
assert_eq!(result.stop, StopReason::ToolUse);
}
}
#[test]
fn hub_config_keys_and_test_backend() {
let t = sse(&[
json!({"type":"message_start","message":{"model":"claude-haiku-5-5","usage":{"input_tokens":3}}}),
json!({"type":"content_block_start","index":0,"content_block":{"type":"text","text":"ready"}}),
json!({"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":1}}),
json!({"type":"message_stop"}),
]);
let models = json!({"data":[{"id":"claude-opus-5-5"},{"id":"claude-haiku-5-5"}]}).to_string();
let (url, seen) = mock(vec![(200, "text/event-stream", t), (200, "application/json", models.clone()), (200, "application/json", models)]);
let (h, d) = hub();
assert!(h.backends().is_empty());
assert!(h.start(None, true).is_err());
add(&h, "Claude", BackendKind::Anthropic, &url, Some("k1"));
add(&h, "Local", BackendKind::OpenaiCompatible, "http://127.0.0.1:9/v1", None);
assert_eq!(h.backends().len(), 2);
assert!(h.has_key("Claude") && h.has_key("Local"));
let r = h.test_backend("Claude").unwrap();
assert_eq!((r.reply.as_str(), r.version_only), ("ready", false));
assert_eq!(r.model, "claude-haiku-5-5");
assert_eq!(seen.lock().unwrap()[0].body["output_config"]["effort"], "high");
assert_eq!(h.list_models("Claude").unwrap(), vec!["claude-opus-5-5", "claude-haiku-5-5"]);
let reach = h.check_reachability();
assert_eq!(reach.len(), 2);
assert!(reach[0].1.is_ok(), "{reach:?}");
assert!(reach[1].1.is_err());
// settings persisted to the file; reload shows both
let s = Settings::load_from(&d.path().join("config.toml"));
assert_eq!(AiConfig::from_settings(&s).backends.len(), 2);
assert!(h.remove_backend("Claude").unwrap());
assert!(!h.remove_backend("Claude").unwrap());
assert!(h.test_backend("Claude").is_err());
// missing key is a clear error
h.add_backend(
BackendConfig {
name: "NoKey".into(),
kind: BackendKind::Openrouter,
model: String::new(),
base_url: Some("http://127.0.0.1:9".into()),
ssh_host: None,
enabled: true,
},
None,
)
.unwrap();
if std::env::var("OPENROUTER_API_KEY").is_err() {
let e = h.start(Some("NoKey"), true).err().unwrap();
assert_eq!(e.code, ErrorCode::BadSettings);
}
assert!(h.compare("Local", "Local", true).is_err());
}
/// #1358: a key saved for an API backend can be replaced or forgotten without removing the
/// backend or its settings, so a rotated key isn't hidden behind an older one.
#[test]
fn a_saved_api_key_can_be_replaced_or_forgotten() {
let (h, _d) = hub();
add(&h, "OpenAI", BackendKind::Openai, "https://api.openai.com/v1", Some("sk-old"));
assert_eq!(h.stored_backend_keys().get("OpenAI"), Some(&true));
assert!(h.saved_backend_key("OpenAI").unwrap());
h.set_backend_key("OpenAI", " sk-new ").unwrap();
assert_eq!(h.keys.get("OpenAI").as_deref(), Some("sk-new"));
assert_eq!(h.key_for(&h.entry("OpenAI").unwrap()).unwrap(), "sk-new", "the replacement is the key a request uses");
h.remove_backend_key("OpenAI").unwrap();
assert_eq!(h.stored_backend_keys().get("OpenAI"), Some(&false));
assert!(!h.saved_backend_key("OpenAI").unwrap());
assert!(h.backends().iter().any(|b| b.name == "OpenAI"), "the backend stays");
assert!(h.set_backend_key("OpenAI", " ").is_err(), "an empty key is refused");
assert!(h.set_backend_key("No such backend", "sk-new").is_err());
assert!(h.remove_backend_key("No such backend").is_err());
add(&h, "Agent", BackendKind::AgentCli, "", None);
assert!(!h.stored_backend_keys().contains_key("Agent"), "an agent app has no saved key here");
assert!(h.saved_backend_key("Agent").is_err());
assert!(h.set_backend_key("Agent", "sk-new").is_err(), "a coding agent keeps its own sign-in");
assert!(h.remove_backend_key("Agent").is_err());
}
/// A key save and another key or backend change wait for each other: a save can't write the key
/// after a remove has deleted the backend and its key.
#[test]
fn a_key_save_waits_for_another_backend_change() {
let (h, _d) = hub();
add(&h, "OpenAI", BackendKind::Openai, "https://api.openai.com/v1", None);
let h = Arc::new(h);
let edit = h.backend_edits.lock().unwrap();
let (started, started_rx) = std::sync::mpsc::channel();
let worker = {
let h = h.clone();
std::thread::spawn(move || {
started.send(()).unwrap();
h.set_backend_key("OpenAI", "sk-new")
})
};
started_rx.recv().unwrap();
// While another change holds the lock, the save can't reach the keychain.
std::thread::sleep(std::time::Duration::from_millis(100));
assert!(!worker.is_finished(), "the save waits for the other backend change");
drop(edit);
worker.join().unwrap().unwrap();
assert_eq!(h.keys.get("OpenAI").as_deref(), Some("sk-new"));
}
/// #369: when OpenRouter has no provider that supports tools for a model (404 once only those are
/// asked for), the turn goes on with the text protocol instead of failing; the first request
/// required parameters, the second sends no tools.
#[test]
fn openrouter_with_no_provider_for_tools_falls_back_to_the_text_protocol() {
use crate::model::{Effort, Msg, Stop, ToolDef, TurnRequest};
let tools = [ToolDef { name: "look".into(), description: "Look at a file".into(), schema: json!({"type":"object"}) }];
let messages = [Msg::user_text("hi")];
let request = TurnRequest { system: "S", messages: &messages, tools: &tools, effort: Effort::Low, max_tokens: 100 };
let none = json!({"error": {"code": 404, "message": "No endpoints found that can handle the requested parameters."}}).to_string();
let reply = chat_sse(&[json!({"choices":[{"delta":{"content":"hello"},"finish_reason":"stop"}]})]);
let (url, seen) = mock(vec![(404, "application/json", none), (200, "text/event-stream", reply)]);
let mut model = openai::OpenAi::openrouter(Some(&url), "k".into(), "moonshotai/kimi-k3");
let result = model.turn(&request, &mut |_| {}, &Stop::default()).unwrap();
assert_eq!(result.message.text(), "hello");
let seen = seen.lock().unwrap();
assert_eq!(seen.len(), 2, "one retry");
assert_eq!(seen[0].body["provider"], json!({"require_parameters": true}));
assert!(seen[0].body.get("tools").is_some());
assert!(seen[1].body.get("provider").is_none() && seen[1].body.get("tools").is_none(), "the second asks in text: {}", seen[1].body);
}
/// #1213: with no max_tokens, OpenRouter holds the model's whole output limit against the key's
/// credit and answers 402 with what it can afford; the request is sent once more capped at that.
#[test]
fn openrouter_asks_again_for_an_answer_the_key_can_afford() {
use crate::model::{Effort, Msg, Stop, TurnRequest};
let messages = [Msg::user_text("hi")];
let request = TurnRequest { system: "S", messages: &messages, tools: &[], effort: Effort::Low, max_tokens: 100 };
let refusal = |n: u64| {
let message = format!(
"This request requires more credits, or fewer max_tokens. You requested up to 131072 tokens, but can only afford {n}. To increase, visit https://openrouter.ai/settings/credits and add more credits"
);
(402, "application/json", json!({"error": {"code": 402, "message": message}}).to_string())
};
let reply = || (200, "text/event-stream", chat_sse(&[json!({"choices":[{"delta":{"content":"hello"},"finish_reason":"stop"}]})]));
let (url, seen) = mock(vec![refusal(35063), reply(), reply()]);
let mut model = openai::OpenAi::openrouter(Some(&url), "k".into(), "moonshotai/kimi-k3.1-pro");
assert_eq!(model.turn(&request, &mut |_| {}, &Stop::default()).unwrap().message.text(), "hello");
assert_eq!(model.turn(&request, &mut |_| {}, &Stop::default()).unwrap().message.text(), "hello");
{
let seen = seen.lock().unwrap();
assert_eq!(seen.len(), 3, "one retry, then the cap is kept");
assert!(seen[0].body.get("max_tokens").is_none());
assert_eq!(seen[1].body["max_tokens"], 35063);
assert_eq!(seen[2].body["max_tokens"], 35063, "the next turn doesn't ask for more again");
}
// Too little left to write a file, or the same refusal under the cap: it stays a billing error.
for (cap, n) in [(None, 1000), (Some(35063), 35063)] {
let (url, seen) = mock(vec![refusal(n), reply()]);
let mut model = openai::OpenAi::openrouter(Some(&url), "k".into(), "moonshotai/kimi-k3.1-pro");
model.answer_cap = cap;
let e = model.turn(&request, &mut |_| {}, &Stop::default()).unwrap_err();
assert!(e.message.contains("402"), "{}", e.message);
assert!(!crate::TurnFailure::from_api(&e).retryable());
assert_eq!(seen.lock().unwrap().len(), 1, "not asked again");
}
// Only OpenRouter is asked again.
let (url, seen) = mock(vec![refusal(35063), reply()]);
let mut other = openai::OpenAi::compatible("Other", &url, Some("k".into()), "m");
assert!(other.turn(&request, &mut |_| {}, &Stop::default()).is_err());
assert_eq!(seen.lock().unwrap().len(), 1);
}
No file is open. Choose one in the tree, or press Ctrl+O (⌘O on a Mac) to find one by name.
Syntax highlighting could not load. The unmodified read-only text remains available above.