merge: resolve conflicts between feat/whatsapp-email-channels and main
- Keep main's WhatsApp implementation (webhook-based, simpler) - Preserve email channel fixes from our branch - Merge all main branch updates (daemon, cron, health, etc.) - Resolve Cargo.lock conflicts
This commit is contained in:
commit
4e6da51924
40 changed files with 6925 additions and 780 deletions
|
|
@ -4,11 +4,13 @@ pub mod gemini;
|
|||
pub mod ollama;
|
||||
pub mod openai;
|
||||
pub mod openrouter;
|
||||
pub mod reliable;
|
||||
pub mod traits;
|
||||
|
||||
pub use traits::Provider;
|
||||
|
||||
use compatible::{AuthStyle, OpenAiCompatibleProvider};
|
||||
use reliable::ReliableProvider;
|
||||
|
||||
/// Factory: create the right provider from config
|
||||
#[allow(clippy::too_many_lines)]
|
||||
|
|
@ -114,6 +116,42 @@ pub fn create_provider(name: &str, api_key: Option<&str>) -> anyhow::Result<Box<
|
|||
}
|
||||
}
|
||||
|
||||
/// Create provider chain with retry and fallback behavior.
|
||||
pub fn create_resilient_provider(
|
||||
primary_name: &str,
|
||||
api_key: Option<&str>,
|
||||
reliability: &crate::config::ReliabilityConfig,
|
||||
) -> anyhow::Result<Box<dyn Provider>> {
|
||||
let mut providers: Vec<(String, Box<dyn Provider>)> = Vec::new();
|
||||
|
||||
providers.push((
|
||||
primary_name.to_string(),
|
||||
create_provider(primary_name, api_key)?,
|
||||
));
|
||||
|
||||
for fallback in &reliability.fallback_providers {
|
||||
if fallback == primary_name || providers.iter().any(|(name, _)| name == fallback) {
|
||||
continue;
|
||||
}
|
||||
|
||||
match create_provider(fallback, api_key) {
|
||||
Ok(provider) => providers.push((fallback.clone(), provider)),
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
fallback_provider = fallback,
|
||||
"Ignoring invalid fallback provider: {e}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Box::new(ReliableProvider::new(
|
||||
providers,
|
||||
reliability.provider_retries,
|
||||
reliability.provider_backoff_ms,
|
||||
)))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
@ -307,6 +345,34 @@ mod tests {
|
|||
assert!(create_provider("", None).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resilient_provider_ignores_duplicate_and_invalid_fallbacks() {
|
||||
let reliability = crate::config::ReliabilityConfig {
|
||||
provider_retries: 1,
|
||||
provider_backoff_ms: 100,
|
||||
fallback_providers: vec![
|
||||
"openrouter".into(),
|
||||
"nonexistent-provider".into(),
|
||||
"openai".into(),
|
||||
"openai".into(),
|
||||
],
|
||||
channel_initial_backoff_secs: 2,
|
||||
channel_max_backoff_secs: 60,
|
||||
scheduler_poll_secs: 15,
|
||||
scheduler_retries: 2,
|
||||
};
|
||||
|
||||
let provider = create_resilient_provider("openrouter", Some("sk-test"), &reliability);
|
||||
assert!(provider.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resilient_provider_errors_for_invalid_primary() {
|
||||
let reliability = crate::config::ReliabilityConfig::default();
|
||||
let provider = create_resilient_provider("totally-invalid", Some("sk-test"), &reliability);
|
||||
assert!(provider.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn factory_all_providers_create_successfully() {
|
||||
let providers = [
|
||||
|
|
|
|||
229
src/providers/reliable.rs
Normal file
229
src/providers/reliable.rs
Normal file
|
|
@ -0,0 +1,229 @@
|
|||
use super::Provider;
|
||||
use async_trait::async_trait;
|
||||
use std::time::Duration;
|
||||
|
||||
/// Provider wrapper with retry + fallback behavior.
|
||||
pub struct ReliableProvider {
|
||||
providers: Vec<(String, Box<dyn Provider>)>,
|
||||
max_retries: u32,
|
||||
base_backoff_ms: u64,
|
||||
}
|
||||
|
||||
impl ReliableProvider {
|
||||
pub fn new(
|
||||
providers: Vec<(String, Box<dyn Provider>)>,
|
||||
max_retries: u32,
|
||||
base_backoff_ms: u64,
|
||||
) -> Self {
|
||||
Self {
|
||||
providers,
|
||||
max_retries,
|
||||
base_backoff_ms: base_backoff_ms.max(50),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Provider for ReliableProvider {
|
||||
async fn chat_with_system(
|
||||
&self,
|
||||
system_prompt: Option<&str>,
|
||||
message: &str,
|
||||
model: &str,
|
||||
temperature: f64,
|
||||
) -> anyhow::Result<String> {
|
||||
let mut failures = Vec::new();
|
||||
|
||||
for (provider_name, provider) in &self.providers {
|
||||
let mut backoff_ms = self.base_backoff_ms;
|
||||
|
||||
for attempt in 0..=self.max_retries {
|
||||
match provider
|
||||
.chat_with_system(system_prompt, message, model, temperature)
|
||||
.await
|
||||
{
|
||||
Ok(resp) => {
|
||||
if attempt > 0 {
|
||||
tracing::info!(
|
||||
provider = provider_name,
|
||||
attempt,
|
||||
"Provider recovered after retries"
|
||||
);
|
||||
}
|
||||
return Ok(resp);
|
||||
}
|
||||
Err(e) => {
|
||||
failures.push(format!(
|
||||
"{provider_name} attempt {}/{}: {e}",
|
||||
attempt + 1,
|
||||
self.max_retries + 1
|
||||
));
|
||||
|
||||
if attempt < self.max_retries {
|
||||
tracing::warn!(
|
||||
provider = provider_name,
|
||||
attempt = attempt + 1,
|
||||
max_retries = self.max_retries,
|
||||
"Provider call failed, retrying"
|
||||
);
|
||||
tokio::time::sleep(Duration::from_millis(backoff_ms)).await;
|
||||
backoff_ms = (backoff_ms.saturating_mul(2)).min(10_000);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tracing::warn!(provider = provider_name, "Switching to fallback provider");
|
||||
}
|
||||
|
||||
anyhow::bail!("All providers failed. Attempts:\n{}", failures.join("\n"))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::Arc;
|
||||
|
||||
struct MockProvider {
|
||||
calls: Arc<AtomicUsize>,
|
||||
fail_until_attempt: usize,
|
||||
response: &'static str,
|
||||
error: &'static str,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Provider for MockProvider {
|
||||
async fn chat_with_system(
|
||||
&self,
|
||||
_system_prompt: Option<&str>,
|
||||
_message: &str,
|
||||
_model: &str,
|
||||
_temperature: f64,
|
||||
) -> anyhow::Result<String> {
|
||||
let attempt = self.calls.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
if attempt <= self.fail_until_attempt {
|
||||
anyhow::bail!(self.error);
|
||||
}
|
||||
Ok(self.response.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn succeeds_without_retry() {
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let provider = ReliableProvider::new(
|
||||
vec![(
|
||||
"primary".into(),
|
||||
Box::new(MockProvider {
|
||||
calls: Arc::clone(&calls),
|
||||
fail_until_attempt: 0,
|
||||
response: "ok",
|
||||
error: "boom",
|
||||
}),
|
||||
)],
|
||||
2,
|
||||
1,
|
||||
);
|
||||
|
||||
let result = provider.chat("hello", "test", 0.0).await.unwrap();
|
||||
assert_eq!(result, "ok");
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn retries_then_recovers() {
|
||||
let calls = Arc::new(AtomicUsize::new(0));
|
||||
let provider = ReliableProvider::new(
|
||||
vec![(
|
||||
"primary".into(),
|
||||
Box::new(MockProvider {
|
||||
calls: Arc::clone(&calls),
|
||||
fail_until_attempt: 1,
|
||||
response: "recovered",
|
||||
error: "temporary",
|
||||
}),
|
||||
)],
|
||||
2,
|
||||
1,
|
||||
);
|
||||
|
||||
let result = provider.chat("hello", "test", 0.0).await.unwrap();
|
||||
assert_eq!(result, "recovered");
|
||||
assert_eq!(calls.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn falls_back_after_retries_exhausted() {
|
||||
let primary_calls = Arc::new(AtomicUsize::new(0));
|
||||
let fallback_calls = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
let provider = ReliableProvider::new(
|
||||
vec![
|
||||
(
|
||||
"primary".into(),
|
||||
Box::new(MockProvider {
|
||||
calls: Arc::clone(&primary_calls),
|
||||
fail_until_attempt: usize::MAX,
|
||||
response: "never",
|
||||
error: "primary down",
|
||||
}),
|
||||
),
|
||||
(
|
||||
"fallback".into(),
|
||||
Box::new(MockProvider {
|
||||
calls: Arc::clone(&fallback_calls),
|
||||
fail_until_attempt: 0,
|
||||
response: "from fallback",
|
||||
error: "fallback down",
|
||||
}),
|
||||
),
|
||||
],
|
||||
1,
|
||||
1,
|
||||
);
|
||||
|
||||
let result = provider.chat("hello", "test", 0.0).await.unwrap();
|
||||
assert_eq!(result, "from fallback");
|
||||
assert_eq!(primary_calls.load(Ordering::SeqCst), 2);
|
||||
assert_eq!(fallback_calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn returns_aggregated_error_when_all_providers_fail() {
|
||||
let provider = ReliableProvider::new(
|
||||
vec![
|
||||
(
|
||||
"p1".into(),
|
||||
Box::new(MockProvider {
|
||||
calls: Arc::new(AtomicUsize::new(0)),
|
||||
fail_until_attempt: usize::MAX,
|
||||
response: "never",
|
||||
error: "p1 error",
|
||||
}),
|
||||
),
|
||||
(
|
||||
"p2".into(),
|
||||
Box::new(MockProvider {
|
||||
calls: Arc::new(AtomicUsize::new(0)),
|
||||
fail_until_attempt: usize::MAX,
|
||||
response: "never",
|
||||
error: "p2 error",
|
||||
}),
|
||||
),
|
||||
],
|
||||
0,
|
||||
1,
|
||||
);
|
||||
|
||||
let err = provider
|
||||
.chat("hello", "test", 0.0)
|
||||
.await
|
||||
.expect_err("all providers should fail");
|
||||
let msg = err.to_string();
|
||||
assert!(msg.contains("All providers failed"));
|
||||
assert!(msg.contains("p1 attempt 1/1"));
|
||||
assert!(msg.contains("p2 attempt 1/1"));
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue