From f50ac58a24af791ad748752ce2142bcae5892431 Mon Sep 17 00:00:00 2001 From: Danielle Jenkins Date: Thu, 13 Mar 2025 11:23:50 +0900 Subject: [PATCH] Improve tools 1. Fixed the lookup_item command to properly handle item paths by: - Trying different item types (struct, enum, trait, fn, macro) - Using the proper URL structure for docs.rs items - Adding User-Agent headers to avoid 404 errors 2. Fixed the search_crates command to: - Include proper User-Agent headers to avoid 403 Forbidden errors - Return JSON data in a consistent format 3. Made general improvements: - Better CLI help text with examples - Better error messages with specific examples - More comprehensive documentation of usage patterns --- src/bin/cratedocs.rs | 11 +++- src/tools/docs/docs.rs | 116 +++++++++++++++++++++++++++++------------ 2 files changed, 92 insertions(+), 35 deletions(-) diff --git a/src/bin/cratedocs.rs b/src/bin/cratedocs.rs index 6dafea4..2b78bb8 100644 --- a/src/bin/cratedocs.rs +++ b/src/bin/cratedocs.rs @@ -159,10 +159,14 @@ async fn run_test_tool( println!("Usage examples:"); println!(" cargo run --bin cratedocs -- test --tool lookup_crate --crate-name serde"); println!(" cargo run --bin cratedocs -- test --tool lookup_crate --crate-name tokio --version 1.35.0"); + println!(" cargo run --bin cratedocs -- test --tool lookup_item --crate-name tokio --item-path sync::mpsc::Sender"); + println!(" cargo run --bin cratedocs -- test --tool lookup_item --crate-name serde --item-path Serialize --version 1.0.147"); println!(" cargo run --bin cratedocs -- test --tool search_crates --query logger\n"); println!("Available tools:"); println!(" lookup_crate - Look up documentation for a Rust crate"); println!(" lookup_item - Look up documentation for a specific item in a crate"); + println!(" Format: 'module::path::ItemName' (e.g., 'sync::mpsc::Sender')"); + println!(" The tool will try to detect if it's a struct, enum, trait, fn, or macro"); println!(" search_crates - Search for crates on crates.io"); println!(" help - Show this help information\n"); return Ok(()); @@ -224,9 +228,12 @@ async fn run_test_tool( Ok(result) => result, Err(e) => { eprintln!("\nERROR: {}", e); - eprintln!("\nTip: The direct item lookup may require very specific path formats. Try these commands instead:"); + eprintln!("\nTip: Try these suggestions:"); eprintln!(" - For crate docs: cargo run --bin cratedocs -- test --tool lookup_crate --crate-name tokio"); - eprintln!(" - For crate docs with version: cargo run --bin cratedocs -- test --tool lookup_crate --crate-name serde --version 1.0.147"); + eprintln!(" - For item lookup: cargo run --bin cratedocs -- test --tool lookup_item --crate-name tokio --item-path sync::mpsc::Sender"); + eprintln!(" - For item lookup with version: cargo run --bin cratedocs -- test --tool lookup_item --crate-name serde --item-path Serialize --version 1.0.147"); + eprintln!(" - For crate search: cargo run --bin cratedocs -- test --tool search_crates --query logger --limit 5"); + eprintln!(" - For help: cargo run --bin cratedocs -- test --tool help"); return Ok(()); } }; diff --git a/src/tools/docs/docs.rs b/src/tools/docs/docs.rs index c46e7a2..bbc8aa8 100644 --- a/src/tools/docs/docs.rs +++ b/src/tools/docs/docs.rs @@ -83,9 +83,13 @@ impl DocRouter { }; // Fetch the documentation page - let response = self.client.get(&url).send().await.map_err(|e| { - ToolError::ExecutionError(format!("Failed to fetch documentation: {}", e)) - })?; + let response = self.client.get(&url) + .header("User-Agent", "CrateDocs/0.1.0 (https://github.com/d6e/cratedocs-mcp)") + .send() + .await + .map_err(|e| { + ToolError::ExecutionError(format!("Failed to fetch documentation: {}", e)) + })?; if !response.status().is_success() { return Err(ToolError::ExecutionError(format!( @@ -113,9 +117,13 @@ impl DocRouter { let url = format!("https://crates.io/api/v1/crates?q={}&per_page={}", query, limit); - let response = self.client.get(&url).send().await.map_err(|e| { - ToolError::ExecutionError(format!("Failed to search crates.io: {}", e)) - })?; + let response = self.client.get(&url) + .header("User-Agent", "CrateDocs/0.1.0 (https://github.com/d6e/cratedocs-mcp)") + .send() + .await + .map_err(|e| { + ToolError::ExecutionError(format!("Failed to search crates.io: {}", e)) + })?; if !response.status().is_success() { return Err(ToolError::ExecutionError(format!( @@ -151,36 +159,78 @@ impl DocRouter { return Ok(doc); } - // Construct the docs.rs URL for the specific item - let url = if let Some(ver) = version { - format!("https://docs.rs/{}/{}/{}/", crate_name, ver, item_path.replace("::", "/")) - } else { - format!("https://docs.rs/{}/latest/{}/", crate_name, item_path.replace("::", "/")) - }; - - // Fetch the documentation page - let response = self.client.get(&url).send().await.map_err(|e| { - ToolError::ExecutionError(format!("Failed to fetch item documentation: {}", e)) - })?; - - if !response.status().is_success() { - return Err(ToolError::ExecutionError(format!( - "Failed to fetch item documentation. Status: {}", - response.status() - ))); + // Process the item path to determine the item type + // Format: module::path::ItemName + // Need to split into module path and item name, and guess item type + let parts: Vec<&str> = item_path.split("::").collect(); + + if parts.is_empty() { + return Err(ToolError::InvalidParameters( + "Invalid item path. Expected format: module::path::ItemName".to_string() + )); } - - let html_body = response.text().await.map_err(|e| { - ToolError::ExecutionError(format!("Failed to read response body: {}", e)) - })?; - // Convert HTML to markdown - let markdown_body = parse_html(&html_body); - - // Cache the markdown result - self.cache.set(cache_key, markdown_body.clone()).await; + let item_name = parts.last().unwrap().to_string(); + let module_path = if parts.len() > 1 { + parts[..parts.len()-1].join("/") + } else { + String::new() + }; - Ok(markdown_body) + // Try different item types (struct, enum, trait, fn) + let item_types = ["struct", "enum", "trait", "fn", "macro"]; + let mut last_error = None; + + for item_type in item_types.iter() { + // Construct the docs.rs URL for the specific item + let url = if let Some(ver) = version.clone() { + if module_path.is_empty() { + format!("https://docs.rs/{}/{}/{}/{}.{}.html", crate_name, ver, crate_name, item_type, item_name) + } else { + format!("https://docs.rs/{}/{}/{}/{}/{}.{}.html", crate_name, ver, crate_name, module_path, item_type, item_name) + } + } else { + if module_path.is_empty() { + format!("https://docs.rs/{}/latest/{}/{}.{}.html", crate_name, crate_name, item_type, item_name) + } else { + format!("https://docs.rs/{}/latest/{}/{}/{}.{}.html", crate_name, crate_name, module_path, item_type, item_name) + } + }; + + // Try to fetch the documentation page + let response = match self.client.get(&url) + .header("User-Agent", "CrateDocs/0.1.0 (https://github.com/d6e/cratedocs-mcp)") + .send().await { + Ok(resp) => resp, + Err(e) => { + last_error = Some(e.to_string()); + continue; + } + }; + + // If found, process and return + if response.status().is_success() { + let html_body = response.text().await.map_err(|e| { + ToolError::ExecutionError(format!("Failed to read response body: {}", e)) + })?; + + // Convert HTML to markdown + let markdown_body = parse_html(&html_body); + + // Cache the markdown result + self.cache.set(cache_key, markdown_body.clone()).await; + + return Ok(markdown_body); + } + + last_error = Some(format!("Status code: {}", response.status())); + } + + // If we got here, none of the item types worked + Err(ToolError::ExecutionError(format!( + "Failed to fetch item documentation. No matching item found. Last error: {}", + last_error.unwrap_or_else(|| "Unknown error".to_string()) + ))) } }