fix: add provider warmup to prevent cold-start timeout on first channel message
The first API request after daemon startup consistently timed out (120s) when using channels (Telegram, Discord, etc.), requiring a retry before succeeding. This happened because the reqwest HTTP client's connection pool was cold — no TLS handshake, DNS resolution, or HTTP/2 negotiation had occurred yet. The fix adds a `warmup()` method to the Provider trait that establishes the connection pool on startup by hitting a lightweight endpoint (`/api/v1/auth/key` for OpenRouter). The channel server calls this immediately after creating the provider, before entering the message processing loop. Tested on Raspberry Pi 5 (aarch64) with OpenRouter + DeepSeek v3.2 via Telegram channel. Before: first message took 2-7 minutes (120s timeout + retries). After: first message responds in <30s with no retries. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
b931aeb56c
commit
cc13fec16d
4 changed files with 37 additions and 0 deletions
|
|
@ -426,6 +426,13 @@ pub async fn start_channels(config: Config) -> Result<()> {
|
||||||
config.api_key.as_deref(),
|
config.api_key.as_deref(),
|
||||||
&config.reliability,
|
&config.reliability,
|
||||||
)?);
|
)?);
|
||||||
|
|
||||||
|
// Warm up the provider connection pool (TLS handshake, DNS, HTTP/2 setup)
|
||||||
|
// so the first real message doesn't hit a cold-start timeout.
|
||||||
|
if let Err(e) = provider.warmup().await {
|
||||||
|
tracing::warn!("Provider warmup failed (non-fatal): {e}");
|
||||||
|
}
|
||||||
|
|
||||||
let model = config
|
let model = config
|
||||||
.default_model
|
.default_model
|
||||||
.clone()
|
.clone()
|
||||||
|
|
|
||||||
|
|
@ -51,6 +51,22 @@ impl OpenRouterProvider {
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl Provider for OpenRouterProvider {
|
impl Provider for OpenRouterProvider {
|
||||||
|
async fn warmup(&self) -> anyhow::Result<()> {
|
||||||
|
// Hit a lightweight endpoint to establish TLS + HTTP/2 connection pool.
|
||||||
|
// This prevents the first real chat request from timing out on cold start.
|
||||||
|
let api_key = self
|
||||||
|
.api_key
|
||||||
|
.as_ref()
|
||||||
|
.ok_or_else(|| anyhow::anyhow!("No API key for warmup"))?;
|
||||||
|
let _ = self
|
||||||
|
.client
|
||||||
|
.get("https://openrouter.ai/api/v1/auth/key")
|
||||||
|
.header("Authorization", format!("Bearer {api_key}"))
|
||||||
|
.send()
|
||||||
|
.await;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
async fn chat_with_system(
|
async fn chat_with_system(
|
||||||
&self,
|
&self,
|
||||||
system_prompt: Option<&str>,
|
system_prompt: Option<&str>,
|
||||||
|
|
|
||||||
|
|
@ -25,6 +25,14 @@ impl ReliableProvider {
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl Provider for ReliableProvider {
|
impl Provider for ReliableProvider {
|
||||||
|
async fn warmup(&self) -> anyhow::Result<()> {
|
||||||
|
if let Some((name, provider)) = self.providers.first() {
|
||||||
|
tracing::info!(provider = name, "Warming up provider connection pool");
|
||||||
|
provider.warmup().await?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
async fn chat_with_system(
|
async fn chat_with_system(
|
||||||
&self,
|
&self,
|
||||||
system_prompt: Option<&str>,
|
system_prompt: Option<&str>,
|
||||||
|
|
|
||||||
|
|
@ -14,4 +14,10 @@ pub trait Provider: Send + Sync {
|
||||||
model: &str,
|
model: &str,
|
||||||
temperature: f64,
|
temperature: f64,
|
||||||
) -> anyhow::Result<String>;
|
) -> anyhow::Result<String>;
|
||||||
|
|
||||||
|
/// Warm up the HTTP connection pool (TLS handshake, DNS, HTTP/2 setup).
|
||||||
|
/// Default implementation is a no-op; providers with HTTP clients should override.
|
||||||
|
async fn warmup(&self) -> anyhow::Result<()> {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue