use crate::approval::{ApprovalManager, ApprovalRequest, ApprovalResponse}; use crate::config::Config; use crate::memory::{self, Memory, MemoryCategory}; use crate::observability::{self, Observer, ObserverEvent}; use crate::providers::{self, ChatMessage, Provider, ToolCall}; use crate::runtime; use crate::security::SecurityPolicy; use crate::tools::{self, Tool}; use crate::util::truncate_with_ellipsis; use anyhow::Result; use regex::{Regex, RegexSet}; use std::fmt::Write; use std::io::Write as _; use std::sync::{Arc, LazyLock}; use std::time::Instant; use uuid::Uuid; /// Minimum characters per chunk when relaying LLM text to a streaming draft. const STREAM_CHUNK_MIN_CHARS: usize = 80; /// Default maximum agentic tool-use iterations per user message to prevent runaway loops. /// Used as a safe fallback when `max_tool_iterations` is unset or configured as zero. const DEFAULT_MAX_TOOL_ITERATIONS: usize = 10; static SENSITIVE_KEY_PATTERNS: LazyLock = LazyLock::new(|| { RegexSet::new([ r"(?i)token", r"(?i)api[_-]?key", r"(?i)password", r"(?i)secret", r"(?i)user[_-]?key", r"(?i)bearer", r"(?i)credential", ]) .unwrap() }); static SENSITIVE_KV_REGEX: LazyLock = LazyLock::new(|| { Regex::new(r#"(?i)(token|api[_-]?key|password|secret|user[_-]?key|bearer|credential)["']?\s*[:=]\s*(?:"([^"]{8,})"|'([^']{8,})'|([a-zA-Z0-9_\-\.]{8,}))"#).unwrap() }); /// Scrub credentials from tool output to prevent accidental exfiltration. /// Replaces known credential patterns with a redacted placeholder while preserving /// a small prefix for context. fn scrub_credentials(input: &str) -> String { SENSITIVE_KV_REGEX .replace_all(input, |caps: ®ex::Captures| { let full_match = &caps[0]; let key = &caps[1]; let val = caps .get(2) .or(caps.get(3)) .or(caps.get(4)) .map(|m| m.as_str()) .unwrap_or(""); // Preserve first 4 chars for context, then redact let prefix = if val.len() > 4 { &val[..4] } else { "" }; if full_match.contains(':') { if full_match.contains('"') { format!("\"{}\": \"{}*[REDACTED]\"", key, prefix) } else { format!("{}: {}*[REDACTED]", key, prefix) } } else if full_match.contains('=') { if full_match.contains('"') { format!("{}=\"{}*[REDACTED]\"", key, prefix) } else { format!("{}={}*[REDACTED]", key, prefix) } } else { format!("{}: {}*[REDACTED]", key, prefix) } }) .to_string() } /// Default trigger for auto-compaction when non-system message count exceeds this threshold. /// Prefer passing the config-driven value via `run_tool_call_loop`; this constant is only /// used when callers omit the parameter. const DEFAULT_MAX_HISTORY_MESSAGES: usize = 50; /// Keep this many most-recent non-system messages after compaction. const COMPACTION_KEEP_RECENT_MESSAGES: usize = 20; /// Safety cap for compaction source transcript passed to the summarizer. const COMPACTION_MAX_SOURCE_CHARS: usize = 12_000; /// Max characters retained in stored compaction summary. const COMPACTION_MAX_SUMMARY_CHARS: usize = 2_000; /// Convert a tool registry to OpenAI function-calling format for native tool support. fn tools_to_openai_format(tools_registry: &[Box]) -> Vec { tools_registry .iter() .map(|tool| { serde_json::json!({ "type": "function", "function": { "name": tool.name(), "description": tool.description(), "parameters": tool.parameters_schema() } }) }) .collect() } fn autosave_memory_key(prefix: &str) -> String { format!("{prefix}_{}", Uuid::new_v4()) } /// Trim conversation history to prevent unbounded growth. /// Preserves the system prompt (first message if role=system) and the most recent messages. fn trim_history(history: &mut Vec, max_history: usize) { // Nothing to trim if within limit let has_system = history.first().map_or(false, |m| m.role == "system"); let non_system_count = if has_system { history.len() - 1 } else { history.len() }; if non_system_count <= max_history { return; } let start = if has_system { 1 } else { 0 }; let to_remove = non_system_count - max_history; history.drain(start..start + to_remove); } fn build_compaction_transcript(messages: &[ChatMessage]) -> String { let mut transcript = String::new(); for msg in messages { let role = msg.role.to_uppercase(); let _ = writeln!(transcript, "{role}: {}", msg.content.trim()); } if transcript.chars().count() > COMPACTION_MAX_SOURCE_CHARS { truncate_with_ellipsis(&transcript, COMPACTION_MAX_SOURCE_CHARS) } else { transcript } } fn apply_compaction_summary( history: &mut Vec, start: usize, compact_end: usize, summary: &str, ) { let summary_msg = ChatMessage::assistant(format!("[Compaction summary]\n{}", summary.trim())); history.splice(start..compact_end, std::iter::once(summary_msg)); } async fn auto_compact_history( history: &mut Vec, provider: &dyn Provider, model: &str, max_history: usize, ) -> Result { let has_system = history.first().map_or(false, |m| m.role == "system"); let non_system_count = if has_system { history.len().saturating_sub(1) } else { history.len() }; if non_system_count <= max_history { return Ok(false); } let start = if has_system { 1 } else { 0 }; let keep_recent = COMPACTION_KEEP_RECENT_MESSAGES.min(non_system_count); let compact_count = non_system_count.saturating_sub(keep_recent); if compact_count == 0 { return Ok(false); } let compact_end = start + compact_count; let to_compact: Vec = history[start..compact_end].to_vec(); let transcript = build_compaction_transcript(&to_compact); let summarizer_system = "You are a conversation compaction engine. Summarize older chat history into concise context for future turns. Preserve: user preferences, commitments, decisions, unresolved tasks, key facts. Omit: filler, repeated chit-chat, verbose tool logs. Output plain text bullet points only."; let summarizer_user = format!( "Summarize the following conversation history for context preservation. Keep it short (max 12 bullet points).\n\n{}", transcript ); let summary_raw = provider .chat_with_system(Some(summarizer_system), &summarizer_user, model, 0.2) .await .unwrap_or_else(|_| { // Fallback to deterministic local truncation when summarization fails. truncate_with_ellipsis(&transcript, COMPACTION_MAX_SUMMARY_CHARS) }); let summary = truncate_with_ellipsis(&summary_raw, COMPACTION_MAX_SUMMARY_CHARS); apply_compaction_summary(history, start, compact_end, &summary); Ok(true) } /// Build context preamble by searching memory for relevant entries. /// Entries with a hybrid score below `min_relevance_score` are dropped to /// prevent unrelated memories from bleeding into the conversation. async fn build_context(mem: &dyn Memory, user_msg: &str, min_relevance_score: f64) -> String { let mut context = String::new(); // Pull relevant memories for this message if let Ok(entries) = mem.recall(user_msg, 5, None).await { let relevant: Vec<_> = entries .iter() .filter(|e| match e.score { Some(score) => score >= min_relevance_score, None => true, }) .collect(); if !relevant.is_empty() { context.push_str("[Memory context]\n"); for entry in &relevant { let _ = writeln!(context, "- {}: {}", entry.key, entry.content); } context.push('\n'); } } context } /// Build hardware datasheet context from RAG when peripherals are enabled. /// Includes pin-alias lookup (e.g. "red_led" → 13) when query matches, plus retrieved chunks. fn build_hardware_context( rag: &crate::rag::HardwareRag, user_msg: &str, boards: &[String], chunk_limit: usize, ) -> String { if rag.is_empty() || boards.is_empty() { return String::new(); } let mut context = String::new(); // Pin aliases: when user says "red led", inject "red_led: 13" for matching boards let pin_ctx = rag.pin_alias_context(user_msg, boards); if !pin_ctx.is_empty() { context.push_str(&pin_ctx); } let chunks = rag.retrieve(user_msg, boards, chunk_limit); if chunks.is_empty() && pin_ctx.is_empty() { return String::new(); } if !chunks.is_empty() { context.push_str("[Hardware documentation]\n"); } for chunk in chunks { let board_tag = chunk.board.as_deref().unwrap_or("generic"); let _ = writeln!( context, "--- {} ({}) ---\n{}\n", chunk.source, board_tag, chunk.content ); } context.push('\n'); context } /// Find a tool by name in the registry. fn find_tool<'a>(tools: &'a [Box], name: &str) -> Option<&'a dyn Tool> { tools.iter().find(|t| t.name() == name).map(|t| t.as_ref()) } fn parse_arguments_value(raw: Option<&serde_json::Value>) -> serde_json::Value { match raw { Some(serde_json::Value::String(s)) => serde_json::from_str::(s) .unwrap_or_else(|_| serde_json::Value::Object(serde_json::Map::new())), Some(value) => value.clone(), None => serde_json::Value::Object(serde_json::Map::new()), } } fn parse_tool_call_value(value: &serde_json::Value) -> Option { if let Some(function) = value.get("function") { let name = function .get("name") .and_then(|v| v.as_str()) .unwrap_or("") .trim() .to_string(); if !name.is_empty() { let arguments = parse_arguments_value(function.get("arguments")); return Some(ParsedToolCall { name, arguments }); } } let name = value .get("name") .and_then(|v| v.as_str()) .unwrap_or("") .trim() .to_string(); if name.is_empty() { return None; } let arguments = parse_arguments_value(value.get("arguments")); Some(ParsedToolCall { name, arguments }) } fn parse_tool_calls_from_json_value(value: &serde_json::Value) -> Vec { let mut calls = Vec::new(); if let Some(tool_calls) = value.get("tool_calls").and_then(|v| v.as_array()) { for call in tool_calls { if let Some(parsed) = parse_tool_call_value(call) { calls.push(parsed); } } if !calls.is_empty() { return calls; } } if let Some(array) = value.as_array() { for item in array { if let Some(parsed) = parse_tool_call_value(item) { calls.push(parsed); } } return calls; } if let Some(parsed) = parse_tool_call_value(value) { calls.push(parsed); } calls } const TOOL_CALL_OPEN_TAGS: [&str; 3] = ["", "", ""]; fn find_first_tag<'a>(haystack: &str, tags: &'a [&'a str]) -> Option<(usize, &'a str)> { tags.iter() .filter_map(|tag| haystack.find(tag).map(|idx| (idx, *tag))) .min_by_key(|(idx, _)| *idx) } fn matching_tool_call_close_tag(open_tag: &str) -> Option<&'static str> { match open_tag { "" => Some(""), "" => Some(""), "" => Some(""), _ => None, } } /// Extract JSON values from a string. /// /// # Security Warning /// /// This function extracts ANY JSON objects/arrays from the input. It MUST only /// be used on content that is already trusted to be from the LLM, such as /// content inside `` tags where the LLM has explicitly indicated intent /// to make a tool call. Do NOT use this on raw user input or content that /// could contain prompt injection payloads. fn extract_json_values(input: &str) -> Vec { let mut values = Vec::new(); let trimmed = input.trim(); if trimmed.is_empty() { return values; } if let Ok(value) = serde_json::from_str::(trimmed) { values.push(value); return values; } let char_positions: Vec<(usize, char)> = trimmed.char_indices().collect(); let mut idx = 0; while idx < char_positions.len() { let (byte_idx, ch) = char_positions[idx]; if ch == '{' || ch == '[' { let slice = &trimmed[byte_idx..]; let mut stream = serde_json::Deserializer::from_str(slice).into_iter::(); if let Some(Ok(value)) = stream.next() { let consumed = stream.byte_offset(); if consumed > 0 { values.push(value); let next_byte = byte_idx + consumed; while idx < char_positions.len() && char_positions[idx].0 < next_byte { idx += 1; } continue; } } } idx += 1; } values } /// Find the end position of a JSON object by tracking balanced braces. fn find_json_end(input: &str) -> Option { let trimmed = input.trim_start(); let offset = input.len() - trimmed.len(); if !trimmed.starts_with('{') { return None; } let mut depth = 0; let mut in_string = false; let mut escape_next = false; for (i, ch) in trimmed.char_indices() { if escape_next { escape_next = false; continue; } match ch { '\\' if in_string => escape_next = true, '"' => in_string = !in_string, '{' if !in_string => depth += 1, '}' if !in_string => { depth -= 1; if depth == 0 { return Some(offset + i + ch.len_utf8()); } } _ => {} } } None } /// Parse GLM-style tool calls from response text. /// GLM uses proprietary formats like: /// - `browser_open/url>https://example.com` /// - `shell/command>ls -la` /// - `http_request/url>https://api.example.com` fn map_glm_tool_alias(tool_name: &str) -> &str { match tool_name { "browser_open" | "browser" | "web_search" | "shell" | "bash" => "shell", "http_request" | "http" => "http_request", _ => tool_name, } } fn build_curl_command(url: &str) -> Option { if !(url.starts_with("http://") || url.starts_with("https://")) { return None; } if url.chars().any(char::is_whitespace) { return None; } let escaped = url.replace('\'', r#"'\\''"#); Some(format!("curl -s '{}'", escaped)) } fn parse_glm_style_tool_calls(text: &str) -> Vec<(String, serde_json::Value, Option)> { let mut calls = Vec::new(); for line in text.lines() { let line = line.trim(); if line.is_empty() { continue; } // Format: tool_name/param>value or tool_name/{json} if let Some(pos) = line.find('/') { let tool_part = &line[..pos]; let rest = &line[pos + 1..]; if tool_part.chars().all(|c| c.is_alphanumeric() || c == '_') { let tool_name = map_glm_tool_alias(tool_part); if let Some(gt_pos) = rest.find('>') { let param_name = rest[..gt_pos].trim(); let value = rest[gt_pos + 1..].trim(); let arguments = match tool_name { "shell" => { if param_name == "url" { let Some(command) = build_curl_command(value) else { continue; }; serde_json::json!({"command": command}) } else if value.starts_with("http://") || value.starts_with("https://") { if let Some(command) = build_curl_command(value) { serde_json::json!({"command": command}) } else { serde_json::json!({"command": value}) } } else { serde_json::json!({"command": value}) } } "http_request" => { serde_json::json!({"url": value, "method": "GET"}) } _ => serde_json::json!({param_name: value}), }; calls.push((tool_name.to_string(), arguments, Some(line.to_string()))); continue; } if rest.starts_with('{') { if let Ok(json_args) = serde_json::from_str::(rest) { calls.push((tool_name.to_string(), json_args, Some(line.to_string()))); } } } } // Plain URL if let Some(command) = build_curl_command(line) { calls.push(( "shell".to_string(), serde_json::json!({"command": command}), Some(line.to_string()), )); } } calls } /// Parse tool calls from an LLM response that uses XML-style function calling. /// /// Expected format (common with system-prompt-guided tool use): /// ```text /// /// {"name": "shell", "arguments": {"command": "ls"}} /// /// ``` /// /// Also accepts common tag variants (``, ``) for model /// compatibility. /// /// Also supports JSON with `tool_calls` array from OpenAI-format responses. fn parse_tool_calls(response: &str) -> (String, Vec) { let mut text_parts = Vec::new(); let mut calls = Vec::new(); let mut remaining = response; // First, try to parse as OpenAI-style JSON response with tool_calls array // This handles providers like Minimax that return tool_calls in native JSON format if let Ok(json_value) = serde_json::from_str::(response.trim()) { calls = parse_tool_calls_from_json_value(&json_value); if !calls.is_empty() { // If we found tool_calls, extract any content field as text if let Some(content) = json_value.get("content").and_then(|v| v.as_str()) { if !content.trim().is_empty() { text_parts.push(content.trim().to_string()); } } return (text_parts.join("\n"), calls); } } // Fall back to XML-style tool-call tag parsing. while let Some((start, open_tag)) = find_first_tag(remaining, &TOOL_CALL_OPEN_TAGS) { // Everything before the tag is text let before = &remaining[..start]; if !before.trim().is_empty() { text_parts.push(before.trim().to_string()); } let Some(close_tag) = matching_tool_call_close_tag(open_tag) else { break; }; let after_open = &remaining[start + open_tag.len()..]; if let Some(close_idx) = after_open.find(close_tag) { let inner = &after_open[..close_idx]; let mut parsed_any = false; let json_values = extract_json_values(inner); for value in json_values { let parsed_calls = parse_tool_calls_from_json_value(&value); if !parsed_calls.is_empty() { parsed_any = true; calls.extend(parsed_calls); } } if !parsed_any { tracing::warn!("Malformed JSON: expected tool-call object in tag body"); } remaining = &after_open[close_idx + close_tag.len()..]; } else { if let Some(json_end) = find_json_end(after_open) { if let Ok(value) = serde_json::from_str::(&after_open[..json_end]) { let parsed_calls = parse_tool_calls_from_json_value(&value); if !parsed_calls.is_empty() { calls.extend(parsed_calls); let after_json = &after_open[json_end..]; if !after_json.trim().is_empty() { text_parts.push(after_json.trim().to_string()); } remaining = ""; } } } break; } } // If XML tags found nothing, try markdown code blocks with tool_call language. // Models behind OpenRouter sometimes output ```tool_call ... ``` or hybrid // ```tool_call ... instead of structured API calls or XML tags. if calls.is_empty() { static MD_TOOL_CALL_RE: LazyLock = LazyLock::new(|| { Regex::new(r"(?s)```tool[_-]?call\s*\n(.*?)(?:```||)") .unwrap() }); let mut md_text_parts: Vec = Vec::new(); let mut last_end = 0; for cap in MD_TOOL_CALL_RE.captures_iter(response) { let full_match = cap.get(0).unwrap(); let before = &response[last_end..full_match.start()]; if !before.trim().is_empty() { md_text_parts.push(before.trim().to_string()); } let inner = &cap[1]; let json_values = extract_json_values(inner); for value in json_values { let parsed_calls = parse_tool_calls_from_json_value(&value); calls.extend(parsed_calls); } last_end = full_match.end(); } if !calls.is_empty() { let after = &response[last_end..]; if !after.trim().is_empty() { md_text_parts.push(after.trim().to_string()); } text_parts = md_text_parts; remaining = ""; } } // GLM-style tool calls (browser_open/url>https://..., shell/command>ls, etc.) if calls.is_empty() { let glm_calls = parse_glm_style_tool_calls(remaining); if !glm_calls.is_empty() { let mut cleaned_text = remaining.to_string(); for (name, args, raw) in &glm_calls { calls.push(ParsedToolCall { name: name.clone(), arguments: args.clone(), }); if let Some(r) = raw { cleaned_text = cleaned_text.replace(r, ""); } } if !cleaned_text.trim().is_empty() { text_parts.push(cleaned_text.trim().to_string()); } remaining = ""; } } // SECURITY: We do NOT fall back to extracting arbitrary JSON from the response // here. That would enable prompt injection attacks where malicious content // (e.g., in emails, files, or web pages) could include JSON that mimics a // tool call. Tool calls MUST be explicitly wrapped in either: // 1. OpenAI-style JSON with a "tool_calls" array // 2. ZeroClaw tool-call tags (, , ) // 3. Markdown code blocks with tool_call/toolcall/tool-call language // 4. Explicit GLM line-based call formats (e.g. `shell/command>...`) // This ensures only the LLM's intentional tool calls are executed. // Remaining text after last tool call if !remaining.trim().is_empty() { text_parts.push(remaining.trim().to_string()); } (text_parts.join("\n"), calls) } fn parse_structured_tool_calls(tool_calls: &[ToolCall]) -> Vec { tool_calls .iter() .map(|call| ParsedToolCall { name: call.name.clone(), arguments: serde_json::from_str::(&call.arguments) .unwrap_or_else(|_| serde_json::Value::Object(serde_json::Map::new())), }) .collect() } /// Build assistant history entry in JSON format for native tool-call APIs. /// `convert_messages` in the OpenRouter provider parses this JSON to reconstruct /// the proper `NativeMessage` with structured `tool_calls`. fn build_native_assistant_history(text: &str, tool_calls: &[ToolCall]) -> String { let calls_json: Vec = tool_calls .iter() .map(|tc| { serde_json::json!({ "id": tc.id, "name": tc.name, "arguments": tc.arguments, }) }) .collect(); let content = if text.trim().is_empty() { serde_json::Value::Null } else { serde_json::Value::String(text.trim().to_string()) }; serde_json::json!({ "content": content, "tool_calls": calls_json, }) .to_string() } fn build_assistant_history_with_tool_calls(text: &str, tool_calls: &[ToolCall]) -> String { let mut parts = Vec::new(); if !text.trim().is_empty() { parts.push(text.trim().to_string()); } for call in tool_calls { let arguments = serde_json::from_str::(&call.arguments) .unwrap_or_else(|_| serde_json::Value::String(call.arguments.clone())); let payload = serde_json::json!({ "id": call.id, "name": call.name, "arguments": arguments, }); parts.push(format!("\n{payload}\n")); } parts.join("\n") } #[derive(Debug)] struct ParsedToolCall { name: String, arguments: serde_json::Value, } /// Execute a single turn of the agent loop: send messages, parse tool calls, /// execute tools, and loop until the LLM produces a final text response. /// When `silent` is true, suppresses stdout (for channel use). #[allow(clippy::too_many_arguments)] pub(crate) async fn agent_turn( provider: &dyn Provider, history: &mut Vec, tools_registry: &[Box], observer: &dyn Observer, provider_name: &str, model: &str, temperature: f64, silent: bool, max_tool_iterations: usize, ) -> Result { run_tool_call_loop( provider, history, tools_registry, observer, provider_name, model, temperature, silent, None, "channel", max_tool_iterations, None, ) .await } /// Execute a single turn of the agent loop: send messages, parse tool calls, /// execute tools, and loop until the LLM produces a final text response. #[allow(clippy::too_many_arguments)] pub(crate) async fn run_tool_call_loop( provider: &dyn Provider, history: &mut Vec, tools_registry: &[Box], observer: &dyn Observer, provider_name: &str, model: &str, temperature: f64, silent: bool, approval: Option<&ApprovalManager>, channel_name: &str, max_tool_iterations: usize, on_delta: Option>, ) -> Result { let max_iterations = if max_tool_iterations == 0 { DEFAULT_MAX_TOOL_ITERATIONS } else { max_tool_iterations }; // Build native tool definitions once if the provider supports them. let use_native_tools = provider.supports_native_tools() && !tools_registry.is_empty(); let tool_definitions = if use_native_tools { tools_to_openai_format(tools_registry) } else { Vec::new() }; for _iteration in 0..max_iterations { observer.record_event(&ObserverEvent::LlmRequest { provider: provider_name.to_string(), model: model.to_string(), messages_count: history.len(), }); let llm_started_at = Instant::now(); // Choose between native tool-call API and prompt-based tool use. // `native_tool_calls` preserves the structured ToolCall vec (with IDs) so // that tool results can later be sent back as proper `role: tool` messages. let (response_text, parsed_text, tool_calls, assistant_history_content, native_tool_calls) = if use_native_tools { match provider .chat_with_tools(history, &tool_definitions, model, temperature) .await { Ok(resp) => { observer.record_event(&ObserverEvent::LlmResponse { provider: provider_name.to_string(), model: model.to_string(), duration: llm_started_at.elapsed(), success: true, error_message: None, }); let response_text = resp.text_or_empty().to_string(); let mut calls = parse_structured_tool_calls(&resp.tool_calls); let mut parsed_text = String::new(); if calls.is_empty() { let (fallback_text, fallback_calls) = parse_tool_calls(&response_text); if !fallback_text.is_empty() { parsed_text = fallback_text; } calls = fallback_calls; } // Use JSON format for native tools so convert_messages() // can reconstruct proper NativeMessage with tool_calls. let assistant_history_content = if resp.tool_calls.is_empty() { response_text.clone() } else { build_native_assistant_history(&response_text, &resp.tool_calls) }; let native_calls = resp.tool_calls; ( response_text, parsed_text, calls, assistant_history_content, native_calls, ) } Err(e) => { observer.record_event(&ObserverEvent::LlmResponse { provider: provider_name.to_string(), model: model.to_string(), duration: llm_started_at.elapsed(), success: false, error_message: Some(crate::providers::sanitize_api_error( &e.to_string(), )), }); return Err(e); } } } else { match provider .chat_with_history(history, model, temperature) .await { Ok(resp) => { observer.record_event(&ObserverEvent::LlmResponse { provider: provider_name.to_string(), model: model.to_string(), duration: llm_started_at.elapsed(), success: true, error_message: None, }); let response_text = resp; let assistant_history_content = response_text.clone(); let (parsed_text, calls) = parse_tool_calls(&response_text); ( response_text, parsed_text, calls, assistant_history_content, Vec::new(), ) } Err(e) => { observer.record_event(&ObserverEvent::LlmResponse { provider: provider_name.to_string(), model: model.to_string(), duration: llm_started_at.elapsed(), success: false, error_message: Some(crate::providers::sanitize_api_error( &e.to_string(), )), }); return Err(e); } } }; let display_text = if parsed_text.is_empty() { response_text.clone() } else { parsed_text }; if tool_calls.is_empty() { // No tool calls — this is the final response. // If a streaming sender is provided, relay the text in small chunks // so the channel can progressively update the draft message. if let Some(ref tx) = on_delta { // Split on whitespace boundaries, accumulating chunks of at least // STREAM_CHUNK_MIN_CHARS characters for progressive draft updates. let mut chunk = String::new(); for word in display_text.split_inclusive(char::is_whitespace) { chunk.push_str(word); if chunk.len() >= STREAM_CHUNK_MIN_CHARS { if tx.send(std::mem::take(&mut chunk)).await.is_err() { break; // receiver dropped } } } if !chunk.is_empty() { let _ = tx.send(chunk).await; } } history.push(ChatMessage::assistant(response_text.clone())); return Ok(display_text); } // Print any text the LLM produced alongside tool calls (unless silent) if !silent && !display_text.is_empty() { print!("{display_text}"); let _ = std::io::stdout().flush(); } // Execute each tool call and build results. // `individual_results` tracks per-call output so that native-mode history // can emit one `role: tool` message per tool call with the correct ID. let mut tool_results = String::new(); let mut individual_results: Vec = Vec::new(); for call in &tool_calls { // ── Approval hook ──────────────────────────────── if let Some(mgr) = approval { if mgr.needs_approval(&call.name) { let request = ApprovalRequest { tool_name: call.name.clone(), arguments: call.arguments.clone(), }; // Only prompt interactively on CLI; auto-approve on other channels. let decision = if channel_name == "cli" { mgr.prompt_cli(&request) } else { ApprovalResponse::Yes }; mgr.record_decision(&call.name, &call.arguments, decision, channel_name); if decision == ApprovalResponse::No { let denied = "Denied by user.".to_string(); individual_results.push(denied.clone()); let _ = writeln!( tool_results, "\n{denied}\n", call.name ); continue; } } } observer.record_event(&ObserverEvent::ToolCallStart { tool: call.name.clone(), }); let start = Instant::now(); let result = if let Some(tool) = find_tool(tools_registry, &call.name) { match tool.execute(call.arguments.clone()).await { Ok(r) => { observer.record_event(&ObserverEvent::ToolCall { tool: call.name.clone(), duration: start.elapsed(), success: r.success, }); if r.success { scrub_credentials(&r.output) } else { format!("Error: {}", r.error.unwrap_or_else(|| r.output)) } } Err(e) => { observer.record_event(&ObserverEvent::ToolCall { tool: call.name.clone(), duration: start.elapsed(), success: false, }); format!("Error executing {}: {e}", call.name) } } } else { format!("Unknown tool: {}", call.name) }; individual_results.push(result.clone()); let _ = writeln!( tool_results, "\n{}\n", call.name, result ); } // Add assistant message with tool calls + tool results to history. // Native mode: use JSON-structured messages so convert_messages() can // reconstruct proper OpenAI-format tool_calls and tool result messages. // Prompt mode: use XML-based text format as before. history.push(ChatMessage::assistant(assistant_history_content)); if native_tool_calls.is_empty() { history.push(ChatMessage::user(format!("[Tool results]\n{tool_results}"))); } else { for (native_call, result) in native_tool_calls.iter().zip(individual_results.iter()) { let tool_msg = serde_json::json!({ "tool_call_id": native_call.id, "content": result, }); history.push(ChatMessage::tool(tool_msg.to_string())); } } } anyhow::bail!("Agent exceeded maximum tool iterations ({max_iterations})") } /// Build the tool instruction block for the system prompt so the LLM knows /// how to invoke tools. pub(crate) fn build_tool_instructions(tools_registry: &[Box]) -> String { let mut instructions = String::new(); instructions.push_str("\n## Tool Use Protocol\n\n"); instructions.push_str("To use a tool, wrap a JSON object in tags:\n\n"); instructions.push_str("```\n\n{\"name\": \"tool_name\", \"arguments\": {\"param\": \"value\"}}\n\n```\n\n"); instructions.push_str( "CRITICAL: Output actual tags—never describe steps or give examples.\n\n", ); instructions.push_str("Example: User says \"what's the date?\". You MUST respond with:\n\n{\"name\":\"shell\",\"arguments\":{\"command\":\"date\"}}\n\n\n"); instructions.push_str("You may use multiple tool calls in a single response. "); instructions.push_str("After tool execution, results appear in tags. "); instructions .push_str("Continue reasoning with the results until you can give a final answer.\n\n"); instructions.push_str("### Available Tools\n\n"); for tool in tools_registry { let _ = writeln!( instructions, "**{}**: {}\nParameters: `{}`\n", tool.name(), tool.description(), tool.parameters_schema() ); } instructions } #[allow(clippy::too_many_lines)] pub async fn run( config: Config, message: Option, provider_override: Option, model_override: Option, temperature: f64, peripheral_overrides: Vec, ) -> Result { // ── Wire up agnostic subsystems ────────────────────────────── let base_observer = observability::create_observer(&config.observability); let observer: Arc = Arc::from(base_observer); let runtime: Arc = Arc::from(runtime::create_runtime(&config.runtime)?); let security = Arc::new(SecurityPolicy::from_config( &config.autonomy, &config.workspace_dir, )); // ── Memory (the brain) ──────────────────────────────────────── let mem: Arc = Arc::from(memory::create_memory( &config.memory, &config.workspace_dir, config.api_key.as_deref(), )?); tracing::info!(backend = mem.name(), "Memory initialized"); // ── Peripherals (merge peripheral tools into registry) ─ if !peripheral_overrides.is_empty() { tracing::info!( peripherals = ?peripheral_overrides, "Peripheral overrides from CLI (config boards take precedence)" ); } // ── Tools (including memory tools and peripherals) ──────────── let (composio_key, composio_entity_id) = if config.composio.enabled { ( config.composio.api_key.as_deref(), Some(config.composio.entity_id.as_str()), ) } else { (None, None) }; let mut tools_registry = tools::all_tools_with_runtime( Arc::new(config.clone()), &security, runtime, mem.clone(), composio_key, composio_entity_id, &config.browser, &config.http_request, &config.workspace_dir, &config.agents, config.api_key.as_deref(), &config, ); let peripheral_tools: Vec> = crate::peripherals::create_peripheral_tools(&config.peripherals).await?; if !peripheral_tools.is_empty() { tracing::info!(count = peripheral_tools.len(), "Peripheral tools added"); tools_registry.extend(peripheral_tools); } // ── Resolve provider ───────────────────────────────────────── let provider_name = provider_override .as_deref() .or(config.default_provider.as_deref()) .unwrap_or("openrouter"); let model_name = model_override .as_deref() .or(config.default_model.as_deref()) .unwrap_or("anthropic/claude-sonnet-4"); let provider: Box = providers::create_routed_provider( provider_name, config.api_key.as_deref(), config.api_url.as_deref(), &config.reliability, &config.model_routes, model_name, )?; observer.record_event(&ObserverEvent::AgentStart { provider: provider_name.to_string(), model: model_name.to_string(), }); // ── Hardware RAG (datasheet retrieval when peripherals + datasheet_dir) ── let hardware_rag: Option = config .peripherals .datasheet_dir .as_ref() .filter(|d| !d.trim().is_empty()) .map(|dir| crate::rag::HardwareRag::load(&config.workspace_dir, dir.trim())) .and_then(Result::ok) .filter(|r: &crate::rag::HardwareRag| !r.is_empty()); if let Some(ref rag) = hardware_rag { tracing::info!(chunks = rag.len(), "Hardware RAG loaded"); } let board_names: Vec = config .peripherals .boards .iter() .map(|b| b.board.clone()) .collect(); // ── Build system prompt from workspace MD files (OpenClaw framework) ── let skills = crate::skills::load_skills(&config.workspace_dir); let mut tool_descs: Vec<(&str, &str)> = vec![ ( "shell", "Execute terminal commands. Use when: running local checks, build/test commands, diagnostics. Don't use when: a safer dedicated tool exists, or command is destructive without approval.", ), ( "file_read", "Read file contents. Use when: inspecting project files, configs, logs. Don't use when: a targeted search is enough.", ), ( "file_write", "Write file contents. Use when: applying focused edits, scaffolding files, updating docs/code. Don't use when: side effects are unclear or file ownership is uncertain.", ), ( "memory_store", "Save to memory. Use when: preserving durable preferences, decisions, key context. Don't use when: information is transient/noisy/sensitive without need.", ), ( "memory_recall", "Search memory. Use when: retrieving prior decisions, user preferences, historical context. Don't use when: answer is already in current context.", ), ( "memory_forget", "Delete a memory entry. Use when: memory is incorrect/stale or explicitly requested for removal. Don't use when: impact is uncertain.", ), ]; tool_descs.push(( "cron_add", "Create a cron job. Supports schedule kinds: cron, at, every; and job types: shell or agent.", )); tool_descs.push(( "cron_list", "List all cron jobs with schedule, status, and metadata.", )); tool_descs.push(("cron_remove", "Remove a cron job by job_id.")); tool_descs.push(( "cron_update", "Patch a cron job (schedule, enabled, command/prompt, model, delivery, session_target).", )); tool_descs.push(( "cron_run", "Force-run a cron job immediately and record a run history entry.", )); tool_descs.push(("cron_runs", "Show recent run history for a cron job.")); tool_descs.push(( "screenshot", "Capture a screenshot of the current screen. Returns file path and base64-encoded PNG. Use when: visual verification, UI inspection, debugging displays.", )); tool_descs.push(( "image_info", "Read image file metadata (format, dimensions, size) and optionally base64-encode it. Use when: inspecting images, preparing visual data for analysis.", )); if config.browser.enabled { tool_descs.push(( "browser_open", "Open approved HTTPS URLs in Brave Browser (allowlist-only, no scraping)", )); } if config.composio.enabled { tool_descs.push(( "composio", "Execute actions on 1000+ apps via Composio (Gmail, Notion, GitHub, Slack, etc.). Use action='list' to discover, 'execute' to run (optionally with connected_account_id), 'connect' to OAuth.", )); } tool_descs.push(( "schedule", "Manage scheduled tasks (create/list/get/cancel/pause/resume). Supports recurring cron and one-shot delays.", )); if !config.agents.is_empty() { tool_descs.push(( "delegate", "Delegate a sub-task to a specialized agent. Use when: task needs different model/capability, or to parallelize work.", )); } if config.peripherals.enabled && !config.peripherals.boards.is_empty() { tool_descs.push(( "gpio_read", "Read GPIO pin value (0 or 1) on connected hardware (STM32, Arduino). Use when: checking sensor/button state, LED status.", )); tool_descs.push(( "gpio_write", "Set GPIO pin high (1) or low (0) on connected hardware. Use when: turning LED on/off, controlling actuators.", )); tool_descs.push(( "arduino_upload", "Upload agent-generated Arduino sketch. Use when: user asks for 'make a heart', 'blink pattern', or custom LED behavior on Arduino. You write the full .ino code; ZeroClaw compiles and uploads it. Pin 13 = built-in LED on Uno.", )); tool_descs.push(( "hardware_memory_map", "Return flash and RAM address ranges for connected hardware. Use when: user asks for 'upper and lower memory addresses', 'memory map', or 'readable addresses'.", )); tool_descs.push(( "hardware_board_info", "Return full board info (chip, architecture, memory map) for connected hardware. Use when: user asks for 'board info', 'what board do I have', 'connected hardware', 'chip info', or 'what hardware'.", )); tool_descs.push(( "hardware_memory_read", "Read actual memory/register values from Nucleo via USB. Use when: user asks to 'read register values', 'read memory', 'dump lower memory 0-126', 'give address and value'. Params: address (hex, default 0x20000000), length (bytes, default 128).", )); tool_descs.push(( "hardware_capabilities", "Query connected hardware for reported GPIO pins and LED pin. Use when: user asks what pins are available.", )); } let bootstrap_max_chars = if config.agent.compact_context { Some(6000) } else { None }; let mut system_prompt = crate::channels::build_system_prompt( &config.workspace_dir, model_name, &tool_descs, &skills, Some(&config.identity), bootstrap_max_chars, ); // Append structured tool-use instructions with schemas system_prompt.push_str(&build_tool_instructions(&tools_registry)); // ── Approval manager (supervised mode) ─────────────────────── let approval_manager = ApprovalManager::from_config(&config.autonomy); // ── Execute ────────────────────────────────────────────────── let start = Instant::now(); let mut final_output = String::new(); if let Some(msg) = message { // Auto-save user message to memory if config.memory.auto_save { let user_key = autosave_memory_key("user_msg"); let _ = mem .store(&user_key, &msg, MemoryCategory::Conversation, None) .await; } // Inject memory + hardware RAG context into user message let mem_context = build_context(mem.as_ref(), &msg, config.memory.min_relevance_score).await; let rag_limit = if config.agent.compact_context { 2 } else { 5 }; let hw_context = hardware_rag .as_ref() .map(|r| build_hardware_context(r, &msg, &board_names, rag_limit)) .unwrap_or_default(); let context = format!("{mem_context}{hw_context}"); let enriched = if context.is_empty() { msg.clone() } else { format!("{context}{msg}") }; let mut history = vec![ ChatMessage::system(&system_prompt), ChatMessage::user(&enriched), ]; let response = run_tool_call_loop( provider.as_ref(), &mut history, &tools_registry, observer.as_ref(), provider_name, model_name, temperature, false, Some(&approval_manager), "cli", config.agent.max_tool_iterations, None, ) .await?; final_output = response.clone(); println!("{response}"); observer.record_event(&ObserverEvent::TurnComplete); // Auto-save assistant response to daily log if config.memory.auto_save { let summary = truncate_with_ellipsis(&response, 100); let response_key = autosave_memory_key("assistant_resp"); let _ = mem .store(&response_key, &summary, MemoryCategory::Daily, None) .await; } } else { println!("🦀 ZeroClaw Interactive Mode"); println!("Type /help for commands.\n"); let cli = crate::channels::CliChannel::new(); // Persistent conversation history across turns let mut history = vec![ChatMessage::system(&system_prompt)]; loop { print!("> "); let _ = std::io::stdout().flush(); let mut input = String::new(); match std::io::stdin().read_line(&mut input) { Ok(0) => break, Ok(_) => {} Err(e) => { eprintln!("\nError reading input: {e}\n"); break; } } let user_input = input.trim().to_string(); if user_input.is_empty() { continue; } match user_input.as_str() { "/quit" | "/exit" => break, "/help" => { println!("Available commands:"); println!(" /help Show this help message"); println!(" /clear /new Clear conversation history"); println!(" /quit /exit Exit interactive mode\n"); continue; } "/clear" | "/new" => { println!( "This will clear the current conversation and delete all session memory." ); println!("Core memories (long-term facts/preferences) will be preserved."); print!("Continue? [y/N] "); let _ = std::io::stdout().flush(); let mut confirm = String::new(); if std::io::stdin().read_line(&mut confirm).is_err() { continue; } if !matches!(confirm.trim().to_lowercase().as_str(), "y" | "yes") { println!("Cancelled.\n"); continue; } history.clear(); history.push(ChatMessage::system(&system_prompt)); // Clear conversation and daily memory let mut cleared = 0; for category in [MemoryCategory::Conversation, MemoryCategory::Daily] { let entries = mem.list(Some(&category), None).await.unwrap_or_default(); for entry in entries { if mem.forget(&entry.key).await.unwrap_or(false) { cleared += 1; } } } if cleared > 0 { println!("Conversation cleared ({cleared} memory entries removed).\n"); } else { println!("Conversation cleared.\n"); } continue; } _ => {} } // Auto-save conversation turns if config.memory.auto_save { let user_key = autosave_memory_key("user_msg"); let _ = mem .store(&user_key, &user_input, MemoryCategory::Conversation, None) .await; } // Inject memory + hardware RAG context into user message let mem_context = build_context(mem.as_ref(), &user_input, config.memory.min_relevance_score).await; let rag_limit = if config.agent.compact_context { 2 } else { 5 }; let hw_context = hardware_rag .as_ref() .map(|r| build_hardware_context(r, &user_input, &board_names, rag_limit)) .unwrap_or_default(); let context = format!("{mem_context}{hw_context}"); let enriched = if context.is_empty() { user_input.clone() } else { format!("{context}{user_input}") }; history.push(ChatMessage::user(&enriched)); let response = match run_tool_call_loop( provider.as_ref(), &mut history, &tools_registry, observer.as_ref(), provider_name, model_name, temperature, false, Some(&approval_manager), "cli", config.agent.max_tool_iterations, None, ) .await { Ok(resp) => resp, Err(e) => { eprintln!("\nError: {e}\n"); continue; } }; final_output = response.clone(); if let Err(e) = crate::channels::Channel::send( &cli, &crate::channels::traits::SendMessage::new(format!("\n{response}\n"), "user"), ) .await { eprintln!("\nError sending CLI response: {e}\n"); } observer.record_event(&ObserverEvent::TurnComplete); // Auto-compaction before hard trimming to preserve long-context signal. if let Ok(compacted) = auto_compact_history( &mut history, provider.as_ref(), model_name, config.agent.max_history_messages, ) .await { if compacted { println!("🧹 Auto-compaction complete"); } } // Hard cap as a safety net. trim_history(&mut history, config.agent.max_history_messages); if config.memory.auto_save { let summary = truncate_with_ellipsis(&response, 100); let response_key = autosave_memory_key("assistant_resp"); let _ = mem .store(&response_key, &summary, MemoryCategory::Daily, None) .await; } } } let duration = start.elapsed(); observer.record_event(&ObserverEvent::AgentEnd { provider: provider_name.to_string(), model: model_name.to_string(), duration, tokens_used: None, cost_usd: None, }); Ok(final_output) } /// Process a single message through the full agent (with tools, peripherals, memory). /// Used by channels (Telegram, Discord, etc.) to enable hardware and tool use. pub async fn process_message(config: Config, message: &str) -> Result { let observer: Arc = Arc::from(observability::create_observer(&config.observability)); let runtime: Arc = Arc::from(runtime::create_runtime(&config.runtime)?); let security = Arc::new(SecurityPolicy::from_config( &config.autonomy, &config.workspace_dir, )); let mem: Arc = Arc::from(memory::create_memory( &config.memory, &config.workspace_dir, config.api_key.as_deref(), )?); let (composio_key, composio_entity_id) = if config.composio.enabled { ( config.composio.api_key.as_deref(), Some(config.composio.entity_id.as_str()), ) } else { (None, None) }; let mut tools_registry = tools::all_tools_with_runtime( Arc::new(config.clone()), &security, runtime, mem.clone(), composio_key, composio_entity_id, &config.browser, &config.http_request, &config.workspace_dir, &config.agents, config.api_key.as_deref(), &config, ); let peripheral_tools: Vec> = crate::peripherals::create_peripheral_tools(&config.peripherals).await?; tools_registry.extend(peripheral_tools); let provider_name = config.default_provider.as_deref().unwrap_or("openrouter"); let model_name = config .default_model .clone() .unwrap_or_else(|| "anthropic/claude-sonnet-4-20250514".into()); let provider: Box = providers::create_routed_provider( provider_name, config.api_key.as_deref(), config.api_url.as_deref(), &config.reliability, &config.model_routes, &model_name, )?; let hardware_rag: Option = config .peripherals .datasheet_dir .as_ref() .filter(|d| !d.trim().is_empty()) .map(|dir| crate::rag::HardwareRag::load(&config.workspace_dir, dir.trim())) .and_then(Result::ok) .filter(|r: &crate::rag::HardwareRag| !r.is_empty()); let board_names: Vec = config .peripherals .boards .iter() .map(|b| b.board.clone()) .collect(); let skills = crate::skills::load_skills(&config.workspace_dir); let mut tool_descs: Vec<(&str, &str)> = vec![ ("shell", "Execute terminal commands."), ("file_read", "Read file contents."), ("file_write", "Write file contents."), ("memory_store", "Save to memory."), ("memory_recall", "Search memory."), ("memory_forget", "Delete a memory entry."), ("screenshot", "Capture a screenshot."), ("image_info", "Read image metadata."), ]; if config.browser.enabled { tool_descs.push(("browser_open", "Open approved URLs in browser.")); } if config.composio.enabled { tool_descs.push(("composio", "Execute actions on 1000+ apps via Composio.")); } if config.peripherals.enabled && !config.peripherals.boards.is_empty() { tool_descs.push(("gpio_read", "Read GPIO pin value on connected hardware.")); tool_descs.push(( "gpio_write", "Set GPIO pin high or low on connected hardware.", )); tool_descs.push(( "arduino_upload", "Upload Arduino sketch. Use for 'make a heart', custom patterns. You write full .ino code; ZeroClaw uploads it.", )); tool_descs.push(( "hardware_memory_map", "Return flash and RAM address ranges. Use when user asks for memory addresses or memory map.", )); tool_descs.push(( "hardware_board_info", "Return full board info (chip, architecture, memory map). Use when user asks for board info, what board, connected hardware, or chip info.", )); tool_descs.push(( "hardware_memory_read", "Read actual memory/register values from Nucleo. Use when user asks to read registers, read memory, dump lower memory 0-126, or give address and value.", )); tool_descs.push(( "hardware_capabilities", "Query connected hardware for reported GPIO pins and LED pin. Use when user asks what pins are available.", )); } let bootstrap_max_chars = if config.agent.compact_context { Some(6000) } else { None }; let mut system_prompt = crate::channels::build_system_prompt( &config.workspace_dir, &model_name, &tool_descs, &skills, Some(&config.identity), bootstrap_max_chars, ); system_prompt.push_str(&build_tool_instructions(&tools_registry)); let mem_context = build_context(mem.as_ref(), message, config.memory.min_relevance_score).await; let rag_limit = if config.agent.compact_context { 2 } else { 5 }; let hw_context = hardware_rag .as_ref() .map(|r| build_hardware_context(r, message, &board_names, rag_limit)) .unwrap_or_default(); let context = format!("{mem_context}{hw_context}"); let enriched = if context.is_empty() { message.to_string() } else { format!("{context}{message}") }; let mut history = vec![ ChatMessage::system(&system_prompt), ChatMessage::user(&enriched), ]; agent_turn( provider.as_ref(), &mut history, &tools_registry, observer.as_ref(), provider_name, &model_name, config.default_temperature, true, config.agent.max_tool_iterations, ) .await } #[cfg(test)] mod tests { use super::*; #[test] fn test_scrub_credentials() { let input = "API_KEY=sk-1234567890abcdef; token: 1234567890; password=\"secret123456\""; let scrubbed = scrub_credentials(input); assert!(scrubbed.contains("API_KEY=sk-1*[REDACTED]")); assert!(scrubbed.contains("token: 1234*[REDACTED]")); assert!(scrubbed.contains("password=\"secr*[REDACTED]\"")); assert!(!scrubbed.contains("abcdef")); assert!(!scrubbed.contains("secret123456")); } #[test] fn test_scrub_credentials_json() { let input = r#"{"api_key": "sk-1234567890", "other": "public"}"#; let scrubbed = scrub_credentials(input); assert!(scrubbed.contains("\"api_key\": \"sk-1*[REDACTED]\"")); assert!(scrubbed.contains("public")); } use crate::memory::{Memory, MemoryCategory, SqliteMemory}; use tempfile::TempDir; #[test] fn parse_tool_calls_extracts_single_call() { let response = r#"Let me check that. {"name": "shell", "arguments": {"command": "ls -la"}} "#; let (text, calls) = parse_tool_calls(response); assert_eq!(text, "Let me check that."); assert_eq!(calls.len(), 1); assert_eq!(calls[0].name, "shell"); assert_eq!( calls[0].arguments.get("command").unwrap().as_str().unwrap(), "ls -la" ); } #[test] fn parse_tool_calls_extracts_multiple_calls() { let response = r#" {"name": "file_read", "arguments": {"path": "a.txt"}} {"name": "file_read", "arguments": {"path": "b.txt"}} "#; let (_, calls) = parse_tool_calls(response); assert_eq!(calls.len(), 2); assert_eq!(calls[0].name, "file_read"); assert_eq!(calls[1].name, "file_read"); } #[test] fn parse_tool_calls_returns_text_only_when_no_calls() { let response = "Just a normal response with no tools."; let (text, calls) = parse_tool_calls(response); assert_eq!(text, "Just a normal response with no tools."); assert!(calls.is_empty()); } #[test] fn parse_tool_calls_handles_malformed_json() { let response = r#" not valid json Some text after."#; let (text, calls) = parse_tool_calls(response); assert!(calls.is_empty()); assert!(text.contains("Some text after.")); } #[test] fn parse_tool_calls_text_before_and_after() { let response = r#"Before text. {"name": "shell", "arguments": {"command": "echo hi"}} After text."#; let (text, calls) = parse_tool_calls(response); assert!(text.contains("Before text.")); assert!(text.contains("After text.")); assert_eq!(calls.len(), 1); } #[test] fn parse_tool_calls_handles_openai_format() { // OpenAI-style response with tool_calls array let response = r#"{"content": "Let me check that for you.", "tool_calls": [{"type": "function", "function": {"name": "shell", "arguments": "{\"command\": \"ls -la\"}"}}]}"#; let (text, calls) = parse_tool_calls(response); assert_eq!(text, "Let me check that for you."); assert_eq!(calls.len(), 1); assert_eq!(calls[0].name, "shell"); assert_eq!( calls[0].arguments.get("command").unwrap().as_str().unwrap(), "ls -la" ); } #[test] fn parse_tool_calls_handles_openai_format_multiple_calls() { let response = r#"{"tool_calls": [{"type": "function", "function": {"name": "file_read", "arguments": "{\"path\": \"a.txt\"}"}}, {"type": "function", "function": {"name": "file_read", "arguments": "{\"path\": \"b.txt\"}"}}]}"#; let (_, calls) = parse_tool_calls(response); assert_eq!(calls.len(), 2); assert_eq!(calls[0].name, "file_read"); assert_eq!(calls[1].name, "file_read"); } #[test] fn parse_tool_calls_openai_format_without_content() { // Some providers don't include content field with tool_calls let response = r#"{"tool_calls": [{"type": "function", "function": {"name": "memory_recall", "arguments": "{}"}}]}"#; let (text, calls) = parse_tool_calls(response); assert!(text.is_empty()); // No content field assert_eq!(calls.len(), 1); assert_eq!(calls[0].name, "memory_recall"); } #[test] fn parse_tool_calls_handles_markdown_json_inside_tool_call_tag() { let response = r#" ```json {"name": "file_write", "arguments": {"path": "test.py", "content": "print('ok')"}} ``` "#; let (text, calls) = parse_tool_calls(response); assert!(text.is_empty()); assert_eq!(calls.len(), 1); assert_eq!(calls[0].name, "file_write"); assert_eq!( calls[0].arguments.get("path").unwrap().as_str().unwrap(), "test.py" ); } #[test] fn parse_tool_calls_handles_noisy_tool_call_tag_body() { let response = r#" I will now call the tool with this payload: {"name": "shell", "arguments": {"command": "pwd"}} "#; let (text, calls) = parse_tool_calls(response); assert!(text.is_empty()); assert_eq!(calls.len(), 1); assert_eq!(calls[0].name, "shell"); assert_eq!( calls[0].arguments.get("command").unwrap().as_str().unwrap(), "pwd" ); } #[test] fn parse_tool_calls_handles_markdown_tool_call_fence() { let response = r#"I'll check that. ```tool_call {"name": "shell", "arguments": {"command": "pwd"}} ``` Done."#; let (text, calls) = parse_tool_calls(response); assert_eq!(calls.len(), 1); assert_eq!(calls[0].name, "shell"); assert_eq!( calls[0].arguments.get("command").unwrap().as_str().unwrap(), "pwd" ); assert!(text.contains("I'll check that.")); assert!(text.contains("Done.")); assert!(!text.contains("```tool_call")); } #[test] fn parse_tool_calls_handles_markdown_tool_call_hybrid_close_tag() { let response = r#"Preface ```tool-call {"name": "shell", "arguments": {"command": "date"}} Tail"#; let (text, calls) = parse_tool_calls(response); assert_eq!(calls.len(), 1); assert_eq!(calls[0].name, "shell"); assert_eq!( calls[0].arguments.get("command").unwrap().as_str().unwrap(), "date" ); assert!(text.contains("Preface")); assert!(text.contains("Tail")); assert!(!text.contains("```tool-call")); } #[test] fn parse_tool_calls_handles_toolcall_tag_alias() { let response = r#" {"name": "shell", "arguments": {"command": "date"}} "#; let (text, calls) = parse_tool_calls(response); assert!(text.is_empty()); assert_eq!(calls.len(), 1); assert_eq!(calls[0].name, "shell"); assert_eq!( calls[0].arguments.get("command").unwrap().as_str().unwrap(), "date" ); } #[test] fn parse_tool_calls_handles_tool_dash_call_tag_alias() { let response = r#" {"name": "shell", "arguments": {"command": "whoami"}} "#; let (text, calls) = parse_tool_calls(response); assert!(text.is_empty()); assert_eq!(calls.len(), 1); assert_eq!(calls[0].name, "shell"); assert_eq!( calls[0].arguments.get("command").unwrap().as_str().unwrap(), "whoami" ); } #[test] fn parse_tool_calls_does_not_cross_match_alias_tags() { let response = r#" {"name": "shell", "arguments": {"command": "date"}} "#; let (text, calls) = parse_tool_calls(response); assert!(calls.is_empty()); assert!(text.contains("")); assert!(text.contains("")); } #[test] fn parse_tool_calls_rejects_raw_tool_json_without_tags() { // SECURITY: Raw JSON without explicit wrappers should NOT be parsed // This prevents prompt injection attacks where malicious content // could include JSON that mimics a tool call. let response = r#"Sure, creating the file now. {"name": "file_write", "arguments": {"path": "hello.py", "content": "print('hello')"}}"#; let (text, calls) = parse_tool_calls(response); assert!(text.contains("Sure, creating the file now.")); assert_eq!( calls.len(), 0, "Raw JSON without wrappers should not be parsed" ); } #[test] fn build_tool_instructions_includes_all_tools() { use crate::security::SecurityPolicy; let security = Arc::new(SecurityPolicy::from_config( &crate::config::AutonomyConfig::default(), std::path::Path::new("/tmp"), )); let tools = tools::default_tools(security); let instructions = build_tool_instructions(&tools); assert!(instructions.contains("## Tool Use Protocol")); assert!(instructions.contains("")); assert!(instructions.contains("shell")); assert!(instructions.contains("file_read")); assert!(instructions.contains("file_write")); } #[test] fn tools_to_openai_format_produces_valid_schema() { use crate::security::SecurityPolicy; let security = Arc::new(SecurityPolicy::from_config( &crate::config::AutonomyConfig::default(), std::path::Path::new("/tmp"), )); let tools = tools::default_tools(security); let formatted = tools_to_openai_format(&tools); assert!(!formatted.is_empty()); for tool_json in &formatted { assert_eq!(tool_json["type"], "function"); assert!(tool_json["function"]["name"].is_string()); assert!(tool_json["function"]["description"].is_string()); assert!(!tool_json["function"]["name"].as_str().unwrap().is_empty()); } // Verify known tools are present let names: Vec<&str> = formatted .iter() .filter_map(|t| t["function"]["name"].as_str()) .collect(); assert!(names.contains(&"shell")); assert!(names.contains(&"file_read")); } #[test] fn trim_history_preserves_system_prompt() { let mut history = vec![ChatMessage::system("system prompt")]; for i in 0..DEFAULT_MAX_HISTORY_MESSAGES + 20 { history.push(ChatMessage::user(format!("msg {i}"))); } let original_len = history.len(); assert!(original_len > DEFAULT_MAX_HISTORY_MESSAGES + 1); trim_history(&mut history, DEFAULT_MAX_HISTORY_MESSAGES); // System prompt preserved assert_eq!(history[0].role, "system"); assert_eq!(history[0].content, "system prompt"); // Trimmed to limit assert_eq!(history.len(), DEFAULT_MAX_HISTORY_MESSAGES + 1); // +1 for system // Most recent messages preserved let last = &history[history.len() - 1]; assert_eq!( last.content, format!("msg {}", DEFAULT_MAX_HISTORY_MESSAGES + 19) ); } #[test] fn trim_history_noop_when_within_limit() { let mut history = vec![ ChatMessage::system("sys"), ChatMessage::user("hello"), ChatMessage::assistant("hi"), ]; trim_history(&mut history, DEFAULT_MAX_HISTORY_MESSAGES); assert_eq!(history.len(), 3); } #[test] fn build_compaction_transcript_formats_roles() { let messages = vec![ ChatMessage::user("I like dark mode"), ChatMessage::assistant("Got it"), ]; let transcript = build_compaction_transcript(&messages); assert!(transcript.contains("USER: I like dark mode")); assert!(transcript.contains("ASSISTANT: Got it")); } #[test] fn apply_compaction_summary_replaces_old_segment() { let mut history = vec![ ChatMessage::system("sys"), ChatMessage::user("old 1"), ChatMessage::assistant("old 2"), ChatMessage::user("recent 1"), ChatMessage::assistant("recent 2"), ]; apply_compaction_summary(&mut history, 1, 3, "- user prefers concise replies"); assert_eq!(history.len(), 4); assert!(history[1].content.contains("Compaction summary")); assert!(history[2].content.contains("recent 1")); assert!(history[3].content.contains("recent 2")); } #[test] fn autosave_memory_key_has_prefix_and_uniqueness() { let key1 = autosave_memory_key("user_msg"); let key2 = autosave_memory_key("user_msg"); assert!(key1.starts_with("user_msg_")); assert!(key2.starts_with("user_msg_")); assert_ne!(key1, key2); } #[tokio::test] async fn autosave_memory_keys_preserve_multiple_turns() { let tmp = TempDir::new().unwrap(); let mem = SqliteMemory::new(tmp.path()).unwrap(); let key1 = autosave_memory_key("user_msg"); let key2 = autosave_memory_key("user_msg"); mem.store(&key1, "I'm Paul", MemoryCategory::Conversation, None) .await .unwrap(); mem.store(&key2, "I'm 45", MemoryCategory::Conversation, None) .await .unwrap(); assert_eq!(mem.count().await.unwrap(), 2); let recalled = mem.recall("45", 5, None).await.unwrap(); assert!(recalled.iter().any(|entry| entry.content.contains("45"))); } // ═══════════════════════════════════════════════════════════════════════ // Recovery Tests - Tool Call Parsing Edge Cases // ═══════════════════════════════════════════════════════════════════════ #[test] fn parse_tool_calls_handles_empty_tool_result() { // Recovery: Empty tool_result tag should be handled gracefully let response = r#"I'll run that command. Done."#; let (text, calls) = parse_tool_calls(response); assert!(text.contains("Done.")); assert!(calls.is_empty()); } #[test] fn parse_arguments_value_handles_null() { // Recovery: null arguments are returned as-is (Value::Null) let value = serde_json::json!(null); let result = parse_arguments_value(Some(&value)); assert!(result.is_null()); } #[test] fn parse_tool_calls_handles_empty_tool_calls_array() { // Recovery: Empty tool_calls array returns original response (no tool parsing) let response = r#"{"content": "Hello", "tool_calls": []}"#; let (text, calls) = parse_tool_calls(response); // When tool_calls is empty, the entire JSON is returned as text assert!(text.contains("Hello")); assert!(calls.is_empty()); } #[test] fn parse_tool_calls_handles_whitespace_only_name() { // Recovery: Whitespace-only tool name should return None let value = serde_json::json!({"function": {"name": " ", "arguments": {}}}); let result = parse_tool_call_value(&value); assert!(result.is_none()); } #[test] fn parse_tool_calls_handles_empty_string_arguments() { // Recovery: Empty string arguments should be handled let value = serde_json::json!({"name": "test", "arguments": ""}); let result = parse_tool_call_value(&value); assert!(result.is_some()); assert_eq!(result.unwrap().name, "test"); } // ═══════════════════════════════════════════════════════════════════════ // Recovery Tests - History Management // ═══════════════════════════════════════════════════════════════════════ #[test] fn trim_history_with_no_system_prompt() { // Recovery: History without system prompt should trim correctly let mut history = vec![]; for i in 0..DEFAULT_MAX_HISTORY_MESSAGES + 20 { history.push(ChatMessage::user(format!("msg {i}"))); } trim_history(&mut history, DEFAULT_MAX_HISTORY_MESSAGES); assert_eq!(history.len(), DEFAULT_MAX_HISTORY_MESSAGES); } #[test] fn trim_history_preserves_role_ordering() { // Recovery: After trimming, role ordering should remain consistent let mut history = vec![ChatMessage::system("system")]; for i in 0..DEFAULT_MAX_HISTORY_MESSAGES + 10 { history.push(ChatMessage::user(format!("user {i}"))); history.push(ChatMessage::assistant(format!("assistant {i}"))); } trim_history(&mut history, DEFAULT_MAX_HISTORY_MESSAGES); assert_eq!(history[0].role, "system"); assert_eq!(history[history.len() - 1].role, "assistant"); } #[test] fn trim_history_with_only_system_prompt() { // Recovery: Only system prompt should not be trimmed let mut history = vec![ChatMessage::system("system prompt")]; trim_history(&mut history, DEFAULT_MAX_HISTORY_MESSAGES); assert_eq!(history.len(), 1); } // ═══════════════════════════════════════════════════════════════════════ // Recovery Tests - Arguments Parsing // ═══════════════════════════════════════════════════════════════════════ #[test] fn parse_arguments_value_handles_invalid_json_string() { // Recovery: Invalid JSON string should return empty object let value = serde_json::Value::String("not valid json".to_string()); let result = parse_arguments_value(Some(&value)); assert!(result.is_object()); assert!(result.as_object().unwrap().is_empty()); } #[test] fn parse_arguments_value_handles_none() { // Recovery: None arguments should return empty object let result = parse_arguments_value(None); assert!(result.is_object()); assert!(result.as_object().unwrap().is_empty()); } // ═══════════════════════════════════════════════════════════════════════ // Recovery Tests - JSON Extraction // ═══════════════════════════════════════════════════════════════════════ #[test] fn extract_json_values_handles_empty_string() { // Recovery: Empty input should return empty vec let result = extract_json_values(""); assert!(result.is_empty()); } #[test] fn extract_json_values_handles_whitespace_only() { // Recovery: Whitespace only should return empty vec let result = extract_json_values(" \n\t "); assert!(result.is_empty()); } #[test] fn extract_json_values_handles_multiple_objects() { // Recovery: Multiple JSON objects should all be extracted let input = r#"{"a": 1}{"b": 2}{"c": 3}"#; let result = extract_json_values(input); assert_eq!(result.len(), 3); } #[test] fn extract_json_values_handles_arrays() { // Recovery: JSON arrays should be extracted let input = r#"[1, 2, 3]{"key": "value"}"#; let result = extract_json_values(input); assert_eq!(result.len(), 2); } // ═══════════════════════════════════════════════════════════════════════ // Recovery Tests - Constants Validation // ═══════════════════════════════════════════════════════════════════════ const _: () = { assert!(DEFAULT_MAX_TOOL_ITERATIONS > 0); assert!(DEFAULT_MAX_TOOL_ITERATIONS <= 100); assert!(DEFAULT_MAX_HISTORY_MESSAGES > 0); assert!(DEFAULT_MAX_HISTORY_MESSAGES <= 1000); }; #[test] fn constants_bounds_are_compile_time_checked() { // Bounds are enforced by the const assertions above. } // ═══════════════════════════════════════════════════════════════════════ // Recovery Tests - Tool Call Value Parsing // ═══════════════════════════════════════════════════════════════════════ #[test] fn parse_tool_call_value_handles_missing_name_field() { // Recovery: Missing name field should return None let value = serde_json::json!({"function": {"arguments": {}}}); let result = parse_tool_call_value(&value); assert!(result.is_none()); } #[test] fn parse_tool_call_value_handles_top_level_name() { // Recovery: Tool call with name at top level (non-OpenAI format) let value = serde_json::json!({"name": "test_tool", "arguments": {}}); let result = parse_tool_call_value(&value); assert!(result.is_some()); assert_eq!(result.unwrap().name, "test_tool"); } #[test] fn parse_tool_calls_from_json_value_handles_empty_array() { // Recovery: Empty tool_calls array should return empty vec let value = serde_json::json!({"tool_calls": []}); let result = parse_tool_calls_from_json_value(&value); assert!(result.is_empty()); } #[test] fn parse_tool_calls_from_json_value_handles_missing_tool_calls() { // Recovery: Missing tool_calls field should fall through let value = serde_json::json!({"name": "test", "arguments": {}}); let result = parse_tool_calls_from_json_value(&value); assert_eq!(result.len(), 1); } #[test] fn parse_tool_calls_from_json_value_handles_top_level_array() { // Recovery: Top-level array of tool calls let value = serde_json::json!([ {"name": "tool_a", "arguments": {}}, {"name": "tool_b", "arguments": {}} ]); let result = parse_tool_calls_from_json_value(&value); assert_eq!(result.len(), 2); } // ═══════════════════════════════════════════════════════════════════════ // GLM-Style Tool Call Parsing // ═══════════════════════════════════════════════════════════════════════ #[test] fn parse_glm_style_browser_open_url() { let response = "browser_open/url>https://example.com"; let calls = parse_glm_style_tool_calls(response); assert_eq!(calls.len(), 1); assert_eq!(calls[0].0, "shell"); assert!(calls[0].1["command"].as_str().unwrap().contains("curl")); assert!(calls[0].1["command"] .as_str() .unwrap() .contains("example.com")); } #[test] fn parse_glm_style_shell_command() { let response = "shell/command>ls -la"; let calls = parse_glm_style_tool_calls(response); assert_eq!(calls.len(), 1); assert_eq!(calls[0].0, "shell"); assert_eq!(calls[0].1["command"], "ls -la"); } #[test] fn parse_glm_style_http_request() { let response = "http_request/url>https://api.example.com/data"; let calls = parse_glm_style_tool_calls(response); assert_eq!(calls.len(), 1); assert_eq!(calls[0].0, "http_request"); assert_eq!(calls[0].1["url"], "https://api.example.com/data"); assert_eq!(calls[0].1["method"], "GET"); } #[test] fn parse_glm_style_plain_url() { let response = "https://example.com/api"; let calls = parse_glm_style_tool_calls(response); assert_eq!(calls.len(), 1); assert_eq!(calls[0].0, "shell"); assert!(calls[0].1["command"].as_str().unwrap().contains("curl")); } #[test] fn parse_glm_style_json_args() { let response = r#"shell/{"command": "echo hello"}"#; let calls = parse_glm_style_tool_calls(response); assert_eq!(calls.len(), 1); assert_eq!(calls[0].0, "shell"); assert_eq!(calls[0].1["command"], "echo hello"); } #[test] fn parse_glm_style_multiple_calls() { let response = r#"shell/command>ls browser_open/url>https://example.com"#; let calls = parse_glm_style_tool_calls(response); assert_eq!(calls.len(), 2); } #[test] fn parse_glm_style_tool_call_integration() { // Integration test: GLM format should be parsed in parse_tool_calls let response = "Checking...\nbrowser_open/url>https://example.com\nDone"; let (text, calls) = parse_tool_calls(response); assert_eq!(calls.len(), 1); assert_eq!(calls[0].name, "shell"); assert!(text.contains("Checking")); assert!(text.contains("Done")); } #[test] fn parse_glm_style_rejects_non_http_url_param() { let response = "browser_open/url>javascript:alert(1)"; let calls = parse_glm_style_tool_calls(response); assert!(calls.is_empty()); } #[test] fn parse_tool_calls_handles_unclosed_tool_call_tag() { let response = "{\"name\":\"shell\",\"arguments\":{\"command\":\"pwd\"}}\nDone"; let (text, calls) = parse_tool_calls(response); assert_eq!(calls.len(), 1); assert_eq!(calls[0].name, "shell"); assert_eq!(calls[0].arguments["command"], "pwd"); assert_eq!(text, "Done"); } }