feat(memory): optional SQLite connection open timeout
- Add memory.sqlite_open_timeout_secs config (None = wait indefinitely). - When set, open the DB in a thread with recv_timeout; cap at 300s. - Default remains None for backward compatibility. - Document in README; add tests for timeout path and default.
This commit is contained in:
parent
b3b1679218
commit
73e675d298
5 changed files with 98 additions and 2 deletions
|
|
@ -244,6 +244,9 @@ keyword_weight = 0.3
|
||||||
|
|
||||||
# backend = "none" uses an explicit no-op memory backend (no persistence)
|
# backend = "none" uses an explicit no-op memory backend (no persistence)
|
||||||
|
|
||||||
|
# Optional for backend = "sqlite": max seconds to wait when opening the DB (e.g. file locked). Omit or leave unset for no timeout.
|
||||||
|
# sqlite_open_timeout_secs = 30
|
||||||
|
|
||||||
# Optional for backend = "lucid"
|
# Optional for backend = "lucid"
|
||||||
# ZEROCLAW_LUCID_CMD=/usr/local/bin/lucid # default: lucid
|
# ZEROCLAW_LUCID_CMD=/usr/local/bin/lucid # default: lucid
|
||||||
# ZEROCLAW_LUCID_BUDGET=200 # default: 200
|
# ZEROCLAW_LUCID_BUDGET=200 # default: 200
|
||||||
|
|
|
||||||
|
|
@ -781,6 +781,12 @@ pub struct MemoryConfig {
|
||||||
/// Auto-hydrate from MEMORY_SNAPSHOT.md when brain.db is missing
|
/// Auto-hydrate from MEMORY_SNAPSHOT.md when brain.db is missing
|
||||||
#[serde(default = "default_true")]
|
#[serde(default = "default_true")]
|
||||||
pub auto_hydrate: bool,
|
pub auto_hydrate: bool,
|
||||||
|
|
||||||
|
// ── SQLite backend options ─────────────────────────────────
|
||||||
|
/// For sqlite backend: max seconds to wait when opening the DB (e.g. file locked).
|
||||||
|
/// None = wait indefinitely (default). Recommended max: 300.
|
||||||
|
#[serde(default)]
|
||||||
|
pub sqlite_open_timeout_secs: Option<u64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn default_embedding_provider() -> String {
|
fn default_embedding_provider() -> String {
|
||||||
|
|
@ -845,6 +851,7 @@ impl Default for MemoryConfig {
|
||||||
snapshot_enabled: false,
|
snapshot_enabled: false,
|
||||||
snapshot_on_hygiene: false,
|
snapshot_on_hygiene: false,
|
||||||
auto_hydrate: true,
|
auto_hydrate: true,
|
||||||
|
sqlite_open_timeout_secs: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -2279,6 +2286,7 @@ default_temperature = 0.7
|
||||||
assert_eq!(m.archive_after_days, 7);
|
assert_eq!(m.archive_after_days, 7);
|
||||||
assert_eq!(m.purge_after_days, 30);
|
assert_eq!(m.purge_after_days, 30);
|
||||||
assert_eq!(m.conversation_retention_days, 30);
|
assert_eq!(m.conversation_retention_days, 30);
|
||||||
|
assert!(m.sqlite_open_timeout_secs.is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|
|
||||||
|
|
@ -115,6 +115,7 @@ pub fn create_memory(
|
||||||
config.vector_weight as f32,
|
config.vector_weight as f32,
|
||||||
config.keyword_weight as f32,
|
config.keyword_weight as f32,
|
||||||
config.embedding_cache_size,
|
config.embedding_cache_size,
|
||||||
|
config.sqlite_open_timeout_secs,
|
||||||
)?;
|
)?;
|
||||||
Ok(mem)
|
Ok(mem)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,14 +1,21 @@
|
||||||
use super::embeddings::EmbeddingProvider;
|
use super::embeddings::EmbeddingProvider;
|
||||||
use super::traits::{Memory, MemoryCategory, MemoryEntry};
|
use super::traits::{Memory, MemoryCategory, MemoryEntry};
|
||||||
use super::vector;
|
use super::vector;
|
||||||
|
use anyhow::Context;
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use chrono::Local;
|
use chrono::Local;
|
||||||
use parking_lot::Mutex;
|
use parking_lot::Mutex;
|
||||||
use rusqlite::{params, Connection};
|
use rusqlite::{params, Connection};
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::sync::mpsc;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
use std::thread;
|
||||||
|
use std::time::Duration;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
/// Maximum allowed open timeout (seconds) to avoid unreasonable waits.
|
||||||
|
const SQLITE_OPEN_TIMEOUT_CAP_SECS: u64 = 300;
|
||||||
|
|
||||||
/// SQLite-backed persistent memory — the brain
|
/// SQLite-backed persistent memory — the brain
|
||||||
///
|
///
|
||||||
/// Full-stack search engine:
|
/// Full-stack search engine:
|
||||||
|
|
@ -34,15 +41,22 @@ impl SqliteMemory {
|
||||||
0.7,
|
0.7,
|
||||||
0.3,
|
0.3,
|
||||||
10_000,
|
10_000,
|
||||||
|
None,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Build SQLite memory with optional open timeout.
|
||||||
|
///
|
||||||
|
/// If `open_timeout_secs` is `Some(n)`, opening the database is limited to `n` seconds
|
||||||
|
/// (capped at 300). Useful when the DB file may be locked or on slow storage.
|
||||||
|
/// `None` = wait indefinitely (default).
|
||||||
pub fn with_embedder(
|
pub fn with_embedder(
|
||||||
workspace_dir: &Path,
|
workspace_dir: &Path,
|
||||||
embedder: Arc<dyn EmbeddingProvider>,
|
embedder: Arc<dyn EmbeddingProvider>,
|
||||||
vector_weight: f32,
|
vector_weight: f32,
|
||||||
keyword_weight: f32,
|
keyword_weight: f32,
|
||||||
cache_max: usize,
|
cache_max: usize,
|
||||||
|
open_timeout_secs: Option<u64>,
|
||||||
) -> anyhow::Result<Self> {
|
) -> anyhow::Result<Self> {
|
||||||
let db_path = workspace_dir.join("memory").join("brain.db");
|
let db_path = workspace_dir.join("memory").join("brain.db");
|
||||||
|
|
||||||
|
|
@ -50,7 +64,7 @@ impl SqliteMemory {
|
||||||
std::fs::create_dir_all(parent)?;
|
std::fs::create_dir_all(parent)?;
|
||||||
}
|
}
|
||||||
|
|
||||||
let conn = Connection::open(&db_path)?;
|
let conn = Self::open_connection(&db_path, open_timeout_secs)?;
|
||||||
|
|
||||||
// ── Production-grade PRAGMA tuning ──────────────────────
|
// ── Production-grade PRAGMA tuning ──────────────────────
|
||||||
// WAL mode: concurrent reads during writes, crash-safe
|
// WAL mode: concurrent reads during writes, crash-safe
|
||||||
|
|
@ -78,6 +92,37 @@ impl SqliteMemory {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Open SQLite connection, optionally with a timeout (for locked/slow storage).
|
||||||
|
fn open_connection(
|
||||||
|
db_path: &Path,
|
||||||
|
open_timeout_secs: Option<u64>,
|
||||||
|
) -> anyhow::Result<Connection> {
|
||||||
|
let path_buf = db_path.to_path_buf();
|
||||||
|
|
||||||
|
let conn = if let Some(secs) = open_timeout_secs {
|
||||||
|
let capped = secs.min(SQLITE_OPEN_TIMEOUT_CAP_SECS);
|
||||||
|
let (tx, rx) = mpsc::channel();
|
||||||
|
thread::spawn(move || {
|
||||||
|
let result = Connection::open(&path_buf);
|
||||||
|
let _ = tx.send(result);
|
||||||
|
});
|
||||||
|
match rx.recv_timeout(Duration::from_secs(capped)) {
|
||||||
|
Ok(Ok(c)) => c,
|
||||||
|
Ok(Err(e)) => return Err(e).context("SQLite failed to open database"),
|
||||||
|
Err(mpsc::RecvTimeoutError::Timeout) => {
|
||||||
|
anyhow::bail!("SQLite connection open timed out after {} seconds", capped);
|
||||||
|
}
|
||||||
|
Err(mpsc::RecvTimeoutError::Disconnected) => {
|
||||||
|
anyhow::bail!("SQLite open thread exited unexpectedly");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
Connection::open(&path_buf).context("SQLite failed to open database")?
|
||||||
|
};
|
||||||
|
|
||||||
|
Ok(conn)
|
||||||
|
}
|
||||||
|
|
||||||
/// Initialize all tables: memories, FTS5, `embedding_cache`
|
/// Initialize all tables: memories, FTS5, `embedding_cache`
|
||||||
fn init_schema(conn: &Connection) -> anyhow::Result<()> {
|
fn init_schema(conn: &Connection) -> anyhow::Result<()> {
|
||||||
conn.execute_batch(
|
conn.execute_batch(
|
||||||
|
|
@ -1054,13 +1099,51 @@ mod tests {
|
||||||
assert_eq!(new, 1);
|
assert_eq!(new, 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Open timeout tests ────────────────────────────────────────
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn open_with_timeout_succeeds_when_fast() {
|
||||||
|
let tmp = TempDir::new().unwrap();
|
||||||
|
let embedder = Arc::new(super::super::embeddings::NoopEmbedding);
|
||||||
|
let mem = SqliteMemory::with_embedder(tmp.path(), embedder, 0.7, 0.3, 1000, Some(5));
|
||||||
|
assert!(
|
||||||
|
mem.is_ok(),
|
||||||
|
"open with 5s timeout should succeed on fast path"
|
||||||
|
);
|
||||||
|
assert_eq!(mem.unwrap().name(), "sqlite");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn open_with_timeout_store_recall_unchanged() {
|
||||||
|
let tmp = TempDir::new().unwrap();
|
||||||
|
let mem = SqliteMemory::with_embedder(
|
||||||
|
tmp.path(),
|
||||||
|
Arc::new(super::super::embeddings::NoopEmbedding),
|
||||||
|
0.7,
|
||||||
|
0.3,
|
||||||
|
1000,
|
||||||
|
Some(2),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
mem.store(
|
||||||
|
"timeout_key",
|
||||||
|
"value with timeout",
|
||||||
|
MemoryCategory::Core,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let entry = mem.get("timeout_key").await.unwrap().unwrap();
|
||||||
|
assert_eq!(entry.content, "value with timeout");
|
||||||
|
}
|
||||||
|
|
||||||
// ── With-embedder constructor test ───────────────────────────
|
// ── With-embedder constructor test ───────────────────────────
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn with_embedder_noop() {
|
fn with_embedder_noop() {
|
||||||
let tmp = TempDir::new().unwrap();
|
let tmp = TempDir::new().unwrap();
|
||||||
let embedder = Arc::new(super::super::embeddings::NoopEmbedding);
|
let embedder = Arc::new(super::super::embeddings::NoopEmbedding);
|
||||||
let mem = SqliteMemory::with_embedder(tmp.path(), embedder, 0.7, 0.3, 1000);
|
let mem = SqliteMemory::with_embedder(tmp.path(), embedder, 0.7, 0.3, 1000, None);
|
||||||
assert!(mem.is_ok());
|
assert!(mem.is_ok());
|
||||||
assert_eq!(mem.unwrap().name(), "sqlite");
|
assert_eq!(mem.unwrap().name(), "sqlite");
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -288,6 +288,7 @@ fn memory_config_defaults_for_backend(backend: &str) -> MemoryConfig {
|
||||||
snapshot_enabled: false,
|
snapshot_enabled: false,
|
||||||
snapshot_on_hygiene: false,
|
snapshot_on_hygiene: false,
|
||||||
auto_hydrate: true,
|
auto_hydrate: true,
|
||||||
|
sqlite_open_timeout_secs: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue