fix(tests): harden brittle tests for cross-platform stability and refactoring resilience

## Problem

The test suite contained several categories of latent brittleness
identified in docs/testing-brittle-tests.md that would surface during
refactoring or cross-platform (Windows) CI execution:

1. Hardcoded Unix paths: \Path::new("/tmp")\ and \PathBuf::from("/tmp")\
   used as workspace directories in agent tests, which fail on Windows
   where /tmp does not exist.

2. Exact string match assertions: ~20 \ssert_eq!(response, "exact text")\
   assertions in agent unit and e2e tests that break on any mock wording
   change, even when the underlying orchestration behavior is correct.

3. Fragile error message string matching: \.contains("specific message")\
   assertions coupled to internal error wording rather than testing the
   error category or behavioral outcome.

## What Changed

### Hardcoded paths → platform-agnostic temp dirs (4 files, 7 locations)
- \src/agent/tests.rs\: Replaced all 4 instances of \Path::new("/tmp")\
  and \PathBuf::from("/tmp")\ with \std::env::temp_dir()\ in
  \make_memory()\, \uild_agent_with()\, \uild_agent_with_memory()\,
  and \uild_agent_with_config()\ helpers.
- \	ests/agent_e2e.rs\: Replaced all 3 instances in \make_memory()\,
  \uild_agent()\, and \uild_agent_xml()\ helpers.

### Exact string assertions → behavioral checks (2 files, ~20 locations)
- \src/agent/tests.rs\: Converted 10 \ssert_eq!(response, "...")\ to
  \ssert!(!response.is_empty(), "descriptive message")\ across tests for
  text pass-through, tool execution, tool failure recovery, XML dispatch,
  mixed text+tool responses, multi-tool batch, and run_single delegation.
- \	ests/agent_e2e.rs\: Converted 9 exact-match assertions to behavioral
  checks. Multi-turn test now uses \ssert_ne!(r1, r2)\ to verify
  sequential responses are distinct without coupling to exact wording.
- Provider error propagation test simplified to \ssert!(result.is_err())\
  without asserting on the error message string.

### Fragile error message assertions → structural checks (2 files)
- \src/tools/git_operations.rs\: Replaced fragile OR-branch string match
  (\contains("git repository") || contains("Git command failed")\) with
  structural assertions: checks \!result.success\, error is non-empty,
  and error does NOT mention autonomy/read-only (verifying the failure
  is git-related, not permission-related).
- \src/cron/scheduler.rs\: Replaced \contains("agent job failed:")\ with
  \!success\ and \!output.is_empty()\ checks that verify failure behavior
  without coupling to exact log format.

## What Was NOT Changed (and why)
- \src/agent/loop_.rs\ parser tests: Exact string assertions are the
  contract for XML tool call parsing — the exact output IS the spec.
- \src/providers/reliable.rs\: Error message assertions test the error
  format contract (provider/model attribution in failure messages).
- \src/service/mod.rs\: Already platform-gated with \#[cfg]\; XML escape
  test is a formatting contract where exact match is appropriate.
- \src/config/schema.rs\: TOML test strings use /tmp as data values for
  deserialization tests, not filesystem access; HOME tests already use
  \std::env::temp_dir()\.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Alex Gorevski 2026-02-17 15:50:19 -08:00 committed by Chummy
parent decea532ed
commit 45cdd25b3d
4 changed files with 38 additions and 32 deletions

View file

@ -255,7 +255,7 @@ fn make_memory() -> Arc<dyn Memory> {
backend: "none".into(),
..MemoryConfig::default()
};
Arc::from(memory::create_memory(&cfg, std::path::Path::new("/tmp"), None).unwrap())
Arc::from(memory::create_memory(&cfg, &std::env::temp_dir(), None).unwrap())
}
fn make_sqlite_memory() -> (Arc<dyn Memory>, tempfile::TempDir) {
@ -283,7 +283,7 @@ fn build_agent_with(
.memory(make_memory())
.observer(make_observer())
.tool_dispatcher(dispatcher)
.workspace_dir(std::path::PathBuf::from("/tmp"))
.workspace_dir(std::env::temp_dir())
.build()
.unwrap()
}
@ -300,7 +300,7 @@ fn build_agent_with_memory(
.memory(mem)
.observer(make_observer())
.tool_dispatcher(Box::new(NativeToolDispatcher))
.workspace_dir(std::path::PathBuf::from("/tmp"))
.workspace_dir(std::env::temp_dir())
.auto_save(auto_save)
.build()
.unwrap()
@ -317,7 +317,7 @@ fn build_agent_with_config(
.memory(make_memory())
.observer(make_observer())
.tool_dispatcher(Box::new(NativeToolDispatcher))
.workspace_dir(std::path::PathBuf::from("/tmp"))
.workspace_dir(std::env::temp_dir())
.config(config)
.build()
.unwrap()
@ -363,7 +363,7 @@ async fn turn_returns_text_when_no_tools_called() {
);
let response = agent.turn("hi").await.unwrap();
assert_eq!(response, "Hello world");
assert!(!response.is_empty(), "Expected non-empty text response from provider");
}
// ═══════════════════════════════════════════════════════════════════════════
@ -388,7 +388,7 @@ async fn turn_executes_single_tool_then_returns() {
);
let response = agent.turn("run echo").await.unwrap();
assert_eq!(response, "I ran the tool");
assert!(!response.is_empty(), "Expected non-empty response after tool execution");
}
// ═══════════════════════════════════════════════════════════════════════════
@ -425,7 +425,7 @@ async fn turn_handles_multi_step_tool_chain() {
);
let response = agent.turn("count 3 times").await.unwrap();
assert_eq!(response, "Done after 3 calls");
assert!(!response.is_empty(), "Expected non-empty response after multi-step chain");
assert_eq!(*count.lock().unwrap(), 3);
}
@ -486,7 +486,7 @@ async fn turn_handles_unknown_tool_gracefully() {
);
let response = agent.turn("use nonexistent").await.unwrap();
assert_eq!(response, "I couldn't find that tool");
assert!(!response.is_empty(), "Expected non-empty response after unknown tool recovery");
// Verify the tool result mentioned "Unknown tool"
let has_tool_result = agent.history().iter().any(|msg| match msg {
@ -523,7 +523,7 @@ async fn turn_recovers_from_tool_failure() {
);
let response = agent.turn("try failing tool").await.unwrap();
assert_eq!(response, "Tool failed but I recovered");
assert!(!response.is_empty(), "Expected non-empty response after tool failure recovery");
}
#[tokio::test]
@ -544,7 +544,7 @@ async fn turn_recovers_from_tool_error() {
);
let response = agent.turn("try panicking").await.unwrap();
assert_eq!(response, "I recovered from the error");
assert!(!response.is_empty(), "Expected non-empty response after tool error recovery");
}
// ═══════════════════════════════════════════════════════════════════════════
@ -560,8 +560,7 @@ async fn turn_propagates_provider_error() {
);
let result = agent.turn("hello").await;
assert!(result.is_err());
assert!(result.unwrap_err().to_string().contains("provider error"));
assert!(result.is_err(), "Expected provider error to propagate");
}
// ═══════════════════════════════════════════════════════════════════════════
@ -666,7 +665,7 @@ async fn xml_dispatcher_parses_and_loops() {
);
let response = agent.turn("test xml").await.unwrap();
assert_eq!(response, "XML tool completed");
assert!(!response.is_empty(), "Expected non-empty response from XML dispatcher");
}
#[tokio::test]
@ -747,7 +746,7 @@ async fn turn_preserves_text_alongside_tool_calls() {
);
let response = agent.turn("check something").await.unwrap();
assert_eq!(response, "Here are the results");
assert!(!response.is_empty(), "Expected non-empty final response after mixed text+tool");
// The intermediate text should be in history
let has_intermediate = agent.history().iter().any(|msg| match msg {
@ -793,7 +792,7 @@ async fn turn_handles_multiple_tools_in_one_response() {
);
let response = agent.turn("batch").await.unwrap();
assert_eq!(response, "All 3 done");
assert!(!response.is_empty(), "Expected non-empty response after multi-tool batch");
assert_eq!(
*count.lock().unwrap(),
3,
@ -1265,5 +1264,5 @@ async fn run_single_delegates_to_turn() {
let mut agent = build_agent_with(provider, vec![], Box::new(NativeToolDispatcher));
let response = agent.run_single("test").await.unwrap();
assert_eq!(response, "via run_single");
assert!(!response.is_empty(), "Expected non-empty response from run_single");
}

View file

@ -642,8 +642,8 @@ mod tests {
job.prompt = Some("Say hello".into());
let (success, output) = run_agent_job(&config, &job).await;
assert!(!success);
assert!(output.contains("agent job failed:"));
assert!(!success, "Agent job without provider key should fail");
assert!(!output.is_empty(), "Expected non-empty error output from failed agent job");
}
#[tokio::test]

View file

@ -758,9 +758,14 @@ mod tests {
// This will fail because there's no git repo, but it shouldn't be blocked by autonomy
let result = tool.execute(json!({"operation": "status"})).await.unwrap();
// The error should be about not being in a git repo, not about read-only mode
// The error should be about git (not about autonomy/read-only mode)
assert!(!result.success, "Expected failure due to missing git repo");
let error_msg = result.error.as_deref().unwrap_or("");
assert!(error_msg.contains("git repository") || error_msg.contains("Git command failed"));
assert!(!error_msg.is_empty(), "Expected a git-related error message");
assert!(
!error_msg.contains("read-only") && !error_msg.contains("autonomy"),
"Error should be about git, not about autonomy restrictions: {error_msg}"
);
}
#[tokio::test]

View file

@ -147,7 +147,7 @@ fn make_memory() -> Arc<dyn Memory> {
backend: "none".into(),
..MemoryConfig::default()
};
Arc::from(memory::create_memory(&cfg, std::path::Path::new("/tmp"), None).unwrap())
Arc::from(memory::create_memory(&cfg, &std::env::temp_dir(), None).unwrap())
}
fn make_observer() -> Arc<dyn Observer> {
@ -175,7 +175,7 @@ fn build_agent(provider: Box<dyn Provider>, tools: Vec<Box<dyn Tool>>) -> Agent
.memory(make_memory())
.observer(make_observer())
.tool_dispatcher(Box::new(NativeToolDispatcher))
.workspace_dir(std::path::PathBuf::from("/tmp"))
.workspace_dir(std::env::temp_dir())
.build()
.unwrap()
}
@ -187,7 +187,7 @@ fn build_agent_xml(provider: Box<dyn Provider>, tools: Vec<Box<dyn Tool>>) -> Ag
.memory(make_memory())
.observer(make_observer())
.tool_dispatcher(Box::new(XmlToolDispatcher))
.workspace_dir(std::path::PathBuf::from("/tmp"))
.workspace_dir(std::env::temp_dir())
.build()
.unwrap()
}
@ -205,7 +205,7 @@ async fn e2e_simple_text_response() {
let mut agent = build_agent(provider, vec![Box::new(EchoTool)]);
let response = agent.turn("hi").await.unwrap();
assert_eq!(response, "Hello from mock provider");
assert!(!response.is_empty(), "Expected non-empty text response");
}
/// Validates single tool call → tool execution → final LLM response.
@ -222,7 +222,7 @@ async fn e2e_single_tool_call_cycle() {
let mut agent = build_agent(provider, vec![Box::new(EchoTool)]);
let response = agent.turn("run echo").await.unwrap();
assert_eq!(response, "Tool executed successfully");
assert!(!response.is_empty(), "Expected non-empty response after tool execution");
}
/// Validates multi-step tool chain: tool A → tool B → tool C → final response.
@ -246,7 +246,7 @@ async fn e2e_multi_step_tool_chain() {
let mut agent = build_agent(provider, vec![Box::new(counting_tool)]);
let response = agent.turn("count twice").await.unwrap();
assert_eq!(response, "Done after 2 tool calls");
assert!(!response.is_empty(), "Expected non-empty response after tool chain");
assert_eq!(*count.lock().unwrap(), 2);
}
@ -268,7 +268,7 @@ async fn e2e_xml_dispatcher_tool_call() {
let mut agent = build_agent_xml(provider, vec![Box::new(EchoTool)]);
let response = agent.turn("test xml dispatch").await.unwrap();
assert_eq!(response, "XML tool executed");
assert!(!response.is_empty(), "Expected non-empty response from XML dispatcher");
}
/// Validates that multiple sequential turns maintain conversation coherence.
@ -283,13 +283,15 @@ async fn e2e_multi_turn_conversation() {
let mut agent = build_agent(provider, vec![Box::new(EchoTool)]);
let r1 = agent.turn("turn 1").await.unwrap();
assert_eq!(r1, "First response");
assert!(!r1.is_empty(), "Expected non-empty first response");
let r2 = agent.turn("turn 2").await.unwrap();
assert_eq!(r2, "Second response");
assert!(!r2.is_empty(), "Expected non-empty second response");
assert_ne!(r1, r2, "Sequential turn responses should be distinct");
let r3 = agent.turn("turn 3").await.unwrap();
assert_eq!(r3, "Third response");
assert!(!r3.is_empty(), "Expected non-empty third response");
assert_ne!(r2, r3, "Sequential turn responses should be distinct");
}
/// Validates that the agent handles unknown tool names gracefully.
@ -306,7 +308,7 @@ async fn e2e_unknown_tool_recovery() {
let mut agent = build_agent(provider, vec![Box::new(EchoTool)]);
let response = agent.turn("call missing tool").await.unwrap();
assert_eq!(response, "Recovered from unknown tool");
assert!(!response.is_empty(), "Expected non-empty response after unknown tool recovery");
}
/// Validates parallel tool dispatch in a single response.
@ -332,6 +334,6 @@ async fn e2e_parallel_tool_dispatch() {
let mut agent = build_agent(provider, vec![Box::new(counting_tool)]);
let response = agent.turn("run both").await.unwrap();
assert_eq!(response, "Both tools ran");
assert!(!response.is_empty(), "Expected non-empty response after parallel dispatch");
assert_eq!(*count.lock().unwrap(), 2);
}