feat: initial release — ZeroClaw v0.1.0
- 22 AI providers (OpenRouter, Anthropic, OpenAI, Mistral, etc.) - 7 channels (CLI, Telegram, Discord, Slack, iMessage, Matrix, Webhook) - 5-step onboarding wizard with Project Context personalization - OpenClaw-aligned system prompt (SOUL.md, IDENTITY.md, USER.md, AGENTS.md, etc.) - SQLite memory backend with auto-save - Skills system with on-demand loading - Security: autonomy levels, command allowlists, cost limits - 532 tests passing, 0 clippy warnings
This commit is contained in:
argenis de la rosa
commit
05cb353f7f
71 changed files with 15757 additions and 0 deletions
369
tests/memory_comparison.rs
Normal file
369
tests/memory_comparison.rs
Normal file
|
|
@ -0,0 +1,369 @@
|
|||
//! Head-to-head comparison: SQLite vs Markdown memory backends
|
||||
//!
|
||||
//! Run with: cargo test --test memory_comparison -- --nocapture
|
||||
|
||||
use std::time::Instant;
|
||||
use tempfile::TempDir;
|
||||
|
||||
// We test both backends through the public memory module
|
||||
use zeroclaw::memory::{
|
||||
markdown::MarkdownMemory, sqlite::SqliteMemory, Memory, MemoryCategory,
|
||||
};
|
||||
|
||||
// ── Helpers ────────────────────────────────────────────────────
|
||||
|
||||
fn sqlite_backend(dir: &std::path::Path) -> SqliteMemory {
|
||||
SqliteMemory::new(dir).expect("SQLite init failed")
|
||||
}
|
||||
|
||||
fn markdown_backend(dir: &std::path::Path) -> MarkdownMemory {
|
||||
MarkdownMemory::new(dir)
|
||||
}
|
||||
|
||||
// ── Test 1: Store performance ──────────────────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn compare_store_speed() {
|
||||
let tmp_sq = TempDir::new().unwrap();
|
||||
let tmp_md = TempDir::new().unwrap();
|
||||
let sq = sqlite_backend(tmp_sq.path());
|
||||
let md = markdown_backend(tmp_md.path());
|
||||
|
||||
let n = 100;
|
||||
|
||||
// SQLite: 100 stores
|
||||
let start = Instant::now();
|
||||
for i in 0..n {
|
||||
sq.store(
|
||||
&format!("key_{i}"),
|
||||
&format!("Memory entry number {i} about Rust programming"),
|
||||
MemoryCategory::Core,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
let sq_dur = start.elapsed();
|
||||
|
||||
// Markdown: 100 stores
|
||||
let start = Instant::now();
|
||||
for i in 0..n {
|
||||
md.store(
|
||||
&format!("key_{i}"),
|
||||
&format!("Memory entry number {i} about Rust programming"),
|
||||
MemoryCategory::Core,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
let md_dur = start.elapsed();
|
||||
|
||||
println!("\n============================================================");
|
||||
println!("STORE {n} entries:");
|
||||
println!(" SQLite: {:?}", sq_dur);
|
||||
println!(" Markdown: {:?}", md_dur);
|
||||
|
||||
// Both should succeed
|
||||
assert_eq!(sq.count().await.unwrap(), n);
|
||||
// Markdown count parses lines, may differ slightly from n
|
||||
let md_count = md.count().await.unwrap();
|
||||
assert!(md_count >= n, "Markdown stored {md_count}, expected >= {n}");
|
||||
}
|
||||
|
||||
// ── Test 2: Recall / search quality ────────────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn compare_recall_quality() {
|
||||
let tmp_sq = TempDir::new().unwrap();
|
||||
let tmp_md = TempDir::new().unwrap();
|
||||
let sq = sqlite_backend(tmp_sq.path());
|
||||
let md = markdown_backend(tmp_md.path());
|
||||
|
||||
// Seed both with identical data
|
||||
let entries = vec![
|
||||
("lang_pref", "User prefers Rust over Python", MemoryCategory::Core),
|
||||
("editor", "Uses VS Code with rust-analyzer", MemoryCategory::Core),
|
||||
("tz", "Timezone is EST, works 9-5", MemoryCategory::Core),
|
||||
("proj1", "Working on ZeroClaw AI assistant", MemoryCategory::Daily),
|
||||
("proj2", "Previous project was a web scraper in Python", MemoryCategory::Daily),
|
||||
("deploy", "Deploys to Hetzner VPS via Docker", MemoryCategory::Core),
|
||||
("model", "Prefers Claude Sonnet for coding tasks", MemoryCategory::Core),
|
||||
("style", "Likes concise responses, no fluff", MemoryCategory::Core),
|
||||
("rust_note", "Rust's ownership model prevents memory bugs", MemoryCategory::Daily),
|
||||
("perf", "Cares about binary size and startup time", MemoryCategory::Core),
|
||||
];
|
||||
|
||||
for (key, content, cat) in &entries {
|
||||
sq.store(key, content, cat.clone()).await.unwrap();
|
||||
md.store(key, content, cat.clone()).await.unwrap();
|
||||
}
|
||||
|
||||
// Test queries and compare results
|
||||
let queries = vec![
|
||||
("Rust", "Should find Rust-related entries"),
|
||||
("Python", "Should find Python references"),
|
||||
("deploy Docker", "Multi-keyword search"),
|
||||
("Claude", "Specific tool reference"),
|
||||
("javascript", "No matches expected"),
|
||||
("binary size startup", "Multi-keyword partial match"),
|
||||
];
|
||||
|
||||
println!("\n============================================================");
|
||||
println!("RECALL QUALITY (10 entries seeded):\n");
|
||||
|
||||
for (query, desc) in &queries {
|
||||
let sq_results = sq.recall(query, 10).await.unwrap();
|
||||
let md_results = md.recall(query, 10).await.unwrap();
|
||||
|
||||
println!(" Query: \"{query}\" — {desc}");
|
||||
println!(" SQLite: {} results", sq_results.len());
|
||||
for r in &sq_results {
|
||||
println!(
|
||||
" [{:.2}] {}: {}",
|
||||
r.score.unwrap_or(0.0),
|
||||
r.key,
|
||||
&r.content[..r.content.len().min(50)]
|
||||
);
|
||||
}
|
||||
println!(" Markdown: {} results", md_results.len());
|
||||
for r in &md_results {
|
||||
println!(
|
||||
" [{:.2}] {}: {}",
|
||||
r.score.unwrap_or(0.0),
|
||||
r.key,
|
||||
&r.content[..r.content.len().min(50)]
|
||||
);
|
||||
}
|
||||
println!();
|
||||
}
|
||||
}
|
||||
|
||||
// ── Test 3: Recall speed at scale ──────────────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn compare_recall_speed() {
|
||||
let tmp_sq = TempDir::new().unwrap();
|
||||
let tmp_md = TempDir::new().unwrap();
|
||||
let sq = sqlite_backend(tmp_sq.path());
|
||||
let md = markdown_backend(tmp_md.path());
|
||||
|
||||
// Seed 200 entries
|
||||
let n = 200;
|
||||
for i in 0..n {
|
||||
let content = if i % 3 == 0 {
|
||||
format!("Rust is great for systems programming, entry {i}")
|
||||
} else if i % 3 == 1 {
|
||||
format!("Python is popular for data science, entry {i}")
|
||||
} else {
|
||||
format!("TypeScript powers modern web apps, entry {i}")
|
||||
};
|
||||
sq.store(&format!("e{i}"), &content, MemoryCategory::Core)
|
||||
.await
|
||||
.unwrap();
|
||||
md.store(&format!("e{i}"), &content, MemoryCategory::Daily)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
// Benchmark recall
|
||||
let start = Instant::now();
|
||||
let sq_results = sq.recall("Rust systems", 10).await.unwrap();
|
||||
let sq_dur = start.elapsed();
|
||||
|
||||
let start = Instant::now();
|
||||
let md_results = md.recall("Rust systems", 10).await.unwrap();
|
||||
let md_dur = start.elapsed();
|
||||
|
||||
println!("\n============================================================");
|
||||
println!("RECALL from {n} entries (query: \"Rust systems\", limit 10):");
|
||||
println!(" SQLite: {:?} → {} results", sq_dur, sq_results.len());
|
||||
println!(" Markdown: {:?} → {} results", md_dur, md_results.len());
|
||||
|
||||
// Both should find results
|
||||
assert!(!sq_results.is_empty());
|
||||
assert!(!md_results.is_empty());
|
||||
}
|
||||
|
||||
// ── Test 4: Persistence (SQLite wins by design) ────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn compare_persistence() {
|
||||
let tmp_sq = TempDir::new().unwrap();
|
||||
let tmp_md = TempDir::new().unwrap();
|
||||
|
||||
// Store in both, then drop and re-open
|
||||
{
|
||||
let sq = sqlite_backend(tmp_sq.path());
|
||||
sq.store("persist_test", "I should survive", MemoryCategory::Core)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
{
|
||||
let md = markdown_backend(tmp_md.path());
|
||||
md.store("persist_test", "I should survive", MemoryCategory::Core)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
// Re-open
|
||||
let sq2 = sqlite_backend(tmp_sq.path());
|
||||
let md2 = markdown_backend(tmp_md.path());
|
||||
|
||||
let sq_entry = sq2.get("persist_test").await.unwrap();
|
||||
let md_entry = md2.get("persist_test").await.unwrap();
|
||||
|
||||
println!("\n============================================================");
|
||||
println!("PERSISTENCE (store → drop → re-open → get):");
|
||||
println!(
|
||||
" SQLite: {}",
|
||||
if sq_entry.is_some() {
|
||||
"✅ Survived"
|
||||
} else {
|
||||
"❌ Lost"
|
||||
}
|
||||
);
|
||||
println!(
|
||||
" Markdown: {}",
|
||||
if md_entry.is_some() {
|
||||
"✅ Survived"
|
||||
} else {
|
||||
"❌ Lost"
|
||||
}
|
||||
);
|
||||
|
||||
// SQLite should always persist by key
|
||||
assert!(sq_entry.is_some());
|
||||
assert_eq!(sq_entry.unwrap().content, "I should survive");
|
||||
|
||||
// Markdown persists content to files (get uses content search)
|
||||
assert!(md_entry.is_some());
|
||||
}
|
||||
|
||||
// ── Test 5: Upsert / update behavior ──────────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn compare_upsert() {
|
||||
let tmp_sq = TempDir::new().unwrap();
|
||||
let tmp_md = TempDir::new().unwrap();
|
||||
let sq = sqlite_backend(tmp_sq.path());
|
||||
let md = markdown_backend(tmp_md.path());
|
||||
|
||||
// Store twice with same key, different content
|
||||
sq.store("pref", "likes Rust", MemoryCategory::Core)
|
||||
.await
|
||||
.unwrap();
|
||||
sq.store("pref", "loves Rust", MemoryCategory::Core)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
md.store("pref", "likes Rust", MemoryCategory::Core)
|
||||
.await
|
||||
.unwrap();
|
||||
md.store("pref", "loves Rust", MemoryCategory::Core)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let sq_count = sq.count().await.unwrap();
|
||||
let md_count = md.count().await.unwrap();
|
||||
|
||||
let sq_entry = sq.get("pref").await.unwrap();
|
||||
let md_results = md.recall("loves Rust", 5).await.unwrap();
|
||||
|
||||
println!("\n============================================================");
|
||||
println!("UPSERT (store same key twice):");
|
||||
println!(" SQLite: count={sq_count}, latest=\"{}\"",
|
||||
sq_entry.as_ref().map_or("none", |e| &e.content));
|
||||
println!(" Markdown: count={md_count} (append-only, both entries kept)");
|
||||
println!(" Can still find latest: {}", !md_results.is_empty());
|
||||
|
||||
// SQLite: upsert replaces, count stays at 1
|
||||
assert_eq!(sq_count, 1);
|
||||
assert_eq!(sq_entry.unwrap().content, "loves Rust");
|
||||
|
||||
// Markdown: append-only, count increases
|
||||
assert!(md_count >= 2, "Markdown should keep both entries");
|
||||
}
|
||||
|
||||
// ── Test 6: Forget / delete capability ─────────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn compare_forget() {
|
||||
let tmp_sq = TempDir::new().unwrap();
|
||||
let tmp_md = TempDir::new().unwrap();
|
||||
let sq = sqlite_backend(tmp_sq.path());
|
||||
let md = markdown_backend(tmp_md.path());
|
||||
|
||||
sq.store("secret", "API key: sk-1234", MemoryCategory::Core)
|
||||
.await
|
||||
.unwrap();
|
||||
md.store("secret", "API key: sk-1234", MemoryCategory::Core)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let sq_forgot = sq.forget("secret").await.unwrap();
|
||||
let md_forgot = md.forget("secret").await.unwrap();
|
||||
|
||||
println!("\n============================================================");
|
||||
println!("FORGET (delete sensitive data):");
|
||||
println!(
|
||||
" SQLite: {} (count={})",
|
||||
if sq_forgot { "✅ Deleted" } else { "❌ Kept" },
|
||||
sq.count().await.unwrap()
|
||||
);
|
||||
println!(
|
||||
" Markdown: {} (append-only by design)",
|
||||
if md_forgot { "✅ Deleted" } else { "⚠️ Cannot delete (audit trail)" },
|
||||
);
|
||||
|
||||
// SQLite can delete
|
||||
assert!(sq_forgot);
|
||||
assert_eq!(sq.count().await.unwrap(), 0);
|
||||
|
||||
// Markdown cannot delete (by design)
|
||||
assert!(!md_forgot);
|
||||
}
|
||||
|
||||
// ── Test 7: Category filtering ─────────────────────────────────
|
||||
|
||||
#[tokio::test]
|
||||
async fn compare_category_filter() {
|
||||
let tmp_sq = TempDir::new().unwrap();
|
||||
let tmp_md = TempDir::new().unwrap();
|
||||
let sq = sqlite_backend(tmp_sq.path());
|
||||
let md = markdown_backend(tmp_md.path());
|
||||
|
||||
// Mix of categories
|
||||
sq.store("a", "core fact 1", MemoryCategory::Core).await.unwrap();
|
||||
sq.store("b", "core fact 2", MemoryCategory::Core).await.unwrap();
|
||||
sq.store("c", "daily note", MemoryCategory::Daily).await.unwrap();
|
||||
sq.store("d", "convo msg", MemoryCategory::Conversation).await.unwrap();
|
||||
|
||||
md.store("a", "core fact 1", MemoryCategory::Core).await.unwrap();
|
||||
md.store("b", "core fact 2", MemoryCategory::Core).await.unwrap();
|
||||
md.store("c", "daily note", MemoryCategory::Daily).await.unwrap();
|
||||
|
||||
let sq_core = sq.list(Some(&MemoryCategory::Core)).await.unwrap();
|
||||
let sq_daily = sq.list(Some(&MemoryCategory::Daily)).await.unwrap();
|
||||
let sq_conv = sq.list(Some(&MemoryCategory::Conversation)).await.unwrap();
|
||||
let sq_all = sq.list(None).await.unwrap();
|
||||
|
||||
let md_core = md.list(Some(&MemoryCategory::Core)).await.unwrap();
|
||||
let md_daily = md.list(Some(&MemoryCategory::Daily)).await.unwrap();
|
||||
let md_all = md.list(None).await.unwrap();
|
||||
|
||||
println!("\n============================================================");
|
||||
println!("CATEGORY FILTERING:");
|
||||
println!(" SQLite: core={}, daily={}, conv={}, all={}",
|
||||
sq_core.len(), sq_daily.len(), sq_conv.len(), sq_all.len());
|
||||
println!(" Markdown: core={}, daily={}, all={}",
|
||||
md_core.len(), md_daily.len(), md_all.len());
|
||||
|
||||
// SQLite: precise category filtering via SQL WHERE
|
||||
assert_eq!(sq_core.len(), 2);
|
||||
assert_eq!(sq_daily.len(), 1);
|
||||
assert_eq!(sq_conv.len(), 1);
|
||||
assert_eq!(sq_all.len(), 4);
|
||||
|
||||
// Markdown: categories determined by file location
|
||||
assert!(!md_core.is_empty());
|
||||
assert!(!md_all.is_empty());
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue