search_tools.py
search_tools.py
"""
Search tools: knowledge base, web search, and file download.
"""
import json
import logging
import os
import time
import traceback
from pathlib import Path
from typing import Union
import requests
from bs4 import BeautifulSoup
from dotenv import load_dotenv
from mcp.types import TextContent
from pydantic import BaseModel, Field
from base import ActionResponse, is_url, download_file_from_url
load_dotenv()
class SearchResult(BaseModel):
"""Individual search result with structured data."""
id: str
title: str
url: str
snippet: str
source: str
class SearchMetadata(BaseModel):
"""Metadata for search operations."""
query: str
search_engine: str
total_results: int
search_time: float | None = None
language: str = "en"
country: str = "us"
async def search_web(
query: str,
num_results: int = 5,
region: str = "wt-wt"
) -> Union[str, TextContent]:
"""
Search the web using DuckDuckGo (free, no API key required).
Args:
query: The search query string
num_results: Number of results to return (1-10)
region: Region code (e.g., 'us-en', 'uk-en', 'wt-wt' for worldwide)
Returns:
TextContent with search results
"""
try:
if not query or not query.strip():
raise ValueError("Search query cannot be empty")
validated_num_results = max(1, min(num_results, 10))
logging.info(f"🔍 Searching for: '{query}'")
start_time = time.time()
# Use DuckDuckGo HTML version for scraping
url = "https://html.duckduckgo.com/html/"
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
}
data = {
"q": query.strip(),
"kl": region
}
response = requests.post(url, data=data, headers=headers, timeout=15)
response.raise_for_status()
search_time = time.time() - start_time
# Parse HTML results
soup = BeautifulSoup(response.text, 'html.parser')
result_divs = soup.find_all('div', class_='result')
search_results = []
for i, result_div in enumerate(result_divs[:validated_num_results]):
try:
# Extract title and URL
title_tag = result_div.find('a', class_='result__a')
if not title_tag:
continue
title = title_tag.get_text(strip=True)
url_link = title_tag.get('href', '')
# Extract snippet
snippet_tag = result_div.find('a', class_='result__snippet')
snippet = snippet_tag.get_text(strip=True) if snippet_tag else ""
result = SearchResult(
id=f"ddg-{i}",
title=title,
url=url_link,
snippet=snippet,
source="duckduckgo"
)
search_results.append(result)
except Exception as e:
logging.warning(f"Error parsing search result {i}: {e}")
continue
metadata = SearchMetadata(
query=query,
search_engine="duckduckgo",
total_results=len(search_results),
search_time=search_time,
language="en",
country=region
)
formatted_content = {
"query": query,
"results": [result.model_dump() for result in search_results],
"count": len(search_results)
}
logging.info(f"✅ Found {len(search_results)} results in {search_time:.2f}s")
action_response = ActionResponse(
success=True,
message=formatted_content,
metadata=metadata.model_dump()
)
return TextContent(
type="text",
text=json.dumps(action_response.model_dump())
)
except Exception as e:
error_msg = f"Search operation failed: {str(e)}"
logging.error(f"Search error: {traceback.format_exc()}")
action_response = ActionResponse(
success=False,
message=error_msg,
metadata={"error_type": "search_error"}
)
return TextContent(
type="text",
text=json.dumps(action_response.model_dump())
)
async def download_file(
url: str,
output_path: str,
overwrite: bool = False,
timeout: int = 180
) -> Union[str, TextContent]:
"""
Download a file from a URL.
Args:
url: URL to download from
output_path: Local path to save the file
overwrite: Whether to overwrite existing files
timeout: Download timeout in seconds
Returns:
TextContent with download result
"""
try:
if not url.startswith(("http://", "https://")):
raise ValueError("Only HTTP/HTTPS URLs are supported")
output_file = Path(output_path).expanduser().resolve()
if output_file.exists() and not overwrite:
raise ValueError(f"File already exists: {output_file}. Use overwrite=True to replace.")
output_file.parent.mkdir(parents=True, exist_ok=True)
logging.info(f"📥 Downloading from: {url}")
start_time = time.time()
temp_path, content = download_file_from_url(url, timeout=timeout)
# Move to final destination
with open(output_file, 'wb') as f:
f.write(content)
# Clean up temp file
Path(temp_path).unlink(missing_ok=True)
duration = time.time() - start_time
file_size = len(content)
logging.info(f"✅ Downloaded {file_size / 1024:.2f} KB in {duration:.2f}s")
action_response = ActionResponse(
success=True,
message=f"Successfully downloaded file to {output_file}",
metadata={
"url": url,
"output_path": str(output_file),
"file_size_bytes": file_size,
"duration_seconds": duration
}
)
return TextContent(
type="text",
text=json.dumps(action_response.model_dump())
)
except Exception as e:
error_msg = f"Download failed: {str(e)}"
logging.error(f"Download error: {traceback.format_exc()}")
action_response = ActionResponse(
success=False,
message=error_msg,
metadata={"error_type": "download_error", "url": url}
)
return TextContent(
type="text",
text=json.dumps(action_response.model_dump())
)
async def search_knowledge_base(
query: str,
knowledge_base_path: str,
top_k: int = 5
) -> Union[str, TextContent]:
"""
Search a local knowledge base using simple text matching.
Args:
query: Search query
knowledge_base_path: Path to knowledge base directory
top_k: Number of top results to return
Returns:
TextContent with search results
"""
try:
kb_path = Path(knowledge_base_path).expanduser().resolve()
if not kb_path.exists():
raise FileNotFoundError(f"Knowledge base not found: {kb_path}")
if not kb_path.is_dir():
raise ValueError(f"Knowledge base path must be a directory: {kb_path}")
logging.info(f"🔍 Searching knowledge base: {kb_path}")
# Simple file search - find files containing the query
results = []
query_lower = query.lower()
for file_path in kb_path.rglob("*"):
if file_path.is_file() and file_path.suffix in [".txt", ".md", ".json"]:
try:
content = file_path.read_text(encoding="utf-8", errors="ignore")
if query_lower in content.lower():
# Get snippet around first occurrence
idx = content.lower().index(query_lower)
start = max(0, idx - 100)
end = min(len(content), idx + 200)
snippet = content[start:end].strip()
results.append({
"file": str(file_path.relative_to(kb_path)),
"snippet": snippet,
"relevance": content.lower().count(query_lower)
})
except Exception as e:
logging.warning(f"Error reading {file_path}: {e}")
continue
# Sort by relevance and limit
results.sort(key=lambda x: x["relevance"], reverse=True)
results = results[:top_k]
logging.info(f"✅ Found {len(results)} results")
action_response = ActionResponse(
success=True,
message={
"query": query,
"results": results,
"total_found": len(results)
},
metadata={
"knowledge_base": str(kb_path),
"top_k": top_k
}
)
return TextContent(
type="text",
text=json.dumps(action_response.model_dump())
)
except Exception as e:
error_msg = f"Knowledge base search failed: {str(e)}"
logging.error(f"KB search error: {traceback.format_exc()}")
action_response = ActionResponse(
success=False,
message=error_msg,
metadata={"error_type": "kb_search_error"}
)
return TextContent(
type="text",
text=json.dumps(action_response.model_dump())
)