◉ClassyGitshadowzzh

tools.py

tools.py
"""Tools for knowledge base interaction"""

import json
import logging
import requests
from typing import Dict, Any, List, Optional
from dataclasses import dataclass
from config import KnowledgeBaseConfig, KnowledgeBaseType


logger = logging.getLogger(__name__)


@dataclass
class SearchResult:
    """Search result from knowledge base"""
    doc_id: str
    chunk_id: str
    text: str
    score: float
    metadata: Dict[str, Any] = None
    
    def to_dict(self) -> Dict[str, Any]:
        return {
            "doc_id": self.doc_id,
            "chunk_id": self.chunk_id,
            "text": self.text,
            "score": self.score,
            "metadata": self.metadata or {}
        }


class KnowledgeBaseTools:
    """Tools for interacting with knowledge base"""
    
    def __init__(self, config: KnowledgeBaseConfig):
        self.config = config
        self.document_store = {}  # In-memory store for documents
        
        # Load document store if exists
        try:
            with open(config.document_store_path, 'r', encoding='utf-8') as f:
                self.document_store = json.load(f)
        except FileNotFoundError:
            logger.info("No existing document store found, starting fresh")
        except Exception as e:
            logger.error(f"Error loading document store: {e}")
    
    def save_document_store(self):
        """Save document store to disk"""
        try:
            with open(self.config.document_store_path, 'w', encoding='utf-8') as f:
                json.dump(self.document_store, f, ensure_ascii=False, indent=2)
        except Exception as e:
            logger.error(f"Error saving document store: {e}")
    
    def knowledge_base_search(self, query: str) -> List[Dict[str, Any]]:
        """
        Search the knowledge base with a natural language query.
        
        Args:
            query: Natural language query string
            
        Returns:
            List of matching document chunks with scores
        """
        try:
            logger.info(f"Knowledge base search initiated - Type: {self.config.type}, Query: '{query}'")
            
            if self.config.type == KnowledgeBaseType.LOCAL:
                return self._search_local(query)
            elif self.config.type == KnowledgeBaseType.DIFY:
                return self._search_dify(query)
            elif self.config.type == KnowledgeBaseType.RAPTOR:
                return self._search_raptor(query)
            elif self.config.type == KnowledgeBaseType.GRAPHRAG:
                return self._search_graphrag(query)
            else:
                logger.error(f"Unsupported knowledge base type: {self.config.type}")
                raise ValueError(f"Unsupported knowledge base type: {self.config.type}")
        except ValueError:
            raise  # Re-raise ValueError for unsupported types
        except Exception as e:
            logger.error(f"Error in knowledge base search: {e}")
            return []
    
    def _search_local(self, query: str) -> List[Dict[str, Any]]:
        """Search using local retrieval pipeline"""
        try:
            logger.info(f"Searching local knowledge base for: {query}")
            
            response = requests.post(
                f"{self.config.local_base_url}/search",
                json={
                    "query": query,
                    "mode": "hybrid",
                    "top_k": self.config.local_top_k,
                    "rerank": True
                },
                timeout=30  # Add timeout
            )
            response.raise_for_status()
            
            results = []
            data = response.json()
            
            # The retrieval pipeline returns results in 'reranked_results' field for hybrid mode
            results_field = data.get("reranked_results") or data.get("results") or []
            
            if not results_field:
                # Check other possible fields
                if data.get("dense_results"):
                    results_field = data.get("dense_results", [])
                elif data.get("sparse_results"):
                    results_field = data.get("sparse_results", [])
                    
                if not results_field:
                    logger.warning(f"Search returned empty results for query: {query}")
                    logger.debug(f"Response keys: {data.keys()}")
                    return []
            
            for item in results_field:
                # Extract doc_id - the field name varies between results
                doc_id = item.get("doc_id", "")
                
                # Use doc_id as chunk_id since the retrieval pipeline indexes chunks
                chunk_id = doc_id
                
                # Get text from the result
                text = item.get("text", "")
                
                # Get score - might be 'rerank_score' or 'score'
                score = item.get("rerank_score", item.get("score", 0.0))
                
                result = SearchResult(
                    doc_id=doc_id,
                    chunk_id=chunk_id,
                    text=text,
                    score=score,
                    metadata=item.get("metadata", {})
                )
                results.append(result.to_dict())
            
            logger.info(f"Local search returned {len(results)} results for query: {query}")
            return results
            
        except requests.exceptions.Timeout as e:
            logger.error(f"Timeout connecting to local retrieval pipeline at {self.config.local_base_url}: {e}")
            return []
        except requests.exceptions.ConnectionError as e:
            logger.error(f"Cannot connect to local retrieval pipeline at {self.config.local_base_url}: {e}")
            logger.info("Make sure the retrieval pipeline is running: cd ../retrieval-pipeline && python main.py")
            return []
        except requests.exceptions.RequestException as e:
            logger.error(f"Error connecting to local retrieval pipeline: {e}")
            return []
    
    def _search_dify(self, query: str) -> List[Dict[str, Any]]:
        """Search using Dify API"""
        if not self.config.dify_api_key:
            logger.error("Dify API key not configured")
            return []
        
        try:
            headers = {
                "Authorization": f"Bearer {self.config.dify_api_key}",
                "Content-Type": "application/json"
            }
            
            payload = {
                "query": query,
                "top_k": self.config.dify_top_k
            }
            
            if self.config.dify_dataset_id:
                payload["dataset_id"] = self.config.dify_dataset_id
            
            response = requests.post(
                f"{self.config.dify_base_url}/datasets/search",
                headers=headers,
                json=payload
            )
            response.raise_for_status()
            
            results = []
            data = response.json()
            
            for item in data.get("data", {}).get("records", []):
                doc_id = item.get("document_id", "")
                chunk_id = item.get("segment_id", f"{doc_id}_chunk_{len(results)}")
                
                result = SearchResult(
                    doc_id=doc_id,
                    chunk_id=chunk_id,
                    text=item.get("content", ""),
                    score=item.get("score", 0.0),
                    metadata=item.get("metadata", {})
                )
                results.append(result.to_dict())
            
            logger.info(f"Dify search returned {len(results)} results")
            return results
            
        except requests.exceptions.RequestException as e:
            logger.error(f"Error connecting to Dify API: {e}")
            return []
    
    def _search_raptor(self, query: str) -> List[Dict[str, Any]]:
        """Search using RAPTOR tree-based index"""
        try:
            response = requests.post(
                f"{self.config.raptor_base_url}/query",
                json={
                    "query": query,
                    "index_type": "raptor",
                    "top_k": self.config.raptor_top_k
                }
            )
            response.raise_for_status()
            
            results = []
            data = response.json()
            
            for i, item in enumerate(data.get("results", [])):
                # RAPTOR returns tree nodes with levels and summaries
                doc_id = item.get("node_id", f"raptor_node_{i}")
                chunk_id = f"{doc_id}_level_{item.get('level', 0)}"
                
                # Use summary if available, otherwise use text
                text_content = item.get("summary", item.get("text", ""))
                
                result = SearchResult(
                    doc_id=doc_id,
                    chunk_id=chunk_id,
                    text=text_content,
                    score=item.get("score", 0.0),
                    metadata={
                        "level": item.get("level", 0),
                        "source": "raptor"
                    }
                )
                results.append(result.to_dict())
            
            logger.info(f"RAPTOR search returned {len(results)} results")
            return results
            
        except requests.exceptions.RequestException as e:
            logger.error(f"Error connecting to RAPTOR index: {e}")
            return []
    
    def _search_graphrag(self, query: str) -> List[Dict[str, Any]]:
        """Search using GraphRAG knowledge graph index"""
        try:
            response = requests.post(
                f"{self.config.graphrag_base_url}/query",
                json={
                    "query": query,
                    "index_type": "graphrag",
                    "top_k": self.config.graphrag_top_k,
                    "search_type": self.config.graphrag_search_type
                }
            )
            response.raise_for_status()
            
            results = []
            data = response.json()
            
            for i, item in enumerate(data.get("results", [])):
                # GraphRAG returns entities or communities
                result_type = item.get("type", "unknown")
                
                if result_type == "entity":
                    doc_id = item.get("id", f"entity_{i}")
                    chunk_id = f"{doc_id}_{item.get('entity_type', 'unknown')}"
                    text_content = f"{item.get('name', '')}. {item.get('description', '')}"
                    metadata = {
                        "type": "entity",
                        "entity_type": item.get("entity_type"),
                        "related_entities": item.get("related_entities", [])
                    }
                else:  # community
                    doc_id = item.get("id", f"community_{i}")
                    chunk_id = f"{doc_id}_level_{item.get('level', 0)}"
                    text_content = item.get("summary", "")
                    metadata = {
                        "type": "community",
                        "level": item.get("level", 0),
                        "entity_count": item.get("entity_count", 0),
                        "sample_entities": item.get("sample_entities", [])
                    }
                
                result = SearchResult(
                    doc_id=doc_id,
                    chunk_id=chunk_id,
                    text=text_content,
                    score=item.get("score", 0.0),
                    metadata={**metadata, "source": "graphrag"}
                )
                results.append(result.to_dict())
            
            logger.info(f"GraphRAG search returned {len(results)} results")
            return results
            
        except requests.exceptions.RequestException as e:
            logger.error(f"Error connecting to GraphRAG index: {e}")
            return []
    
    def get_document(self, doc_id: str) -> Dict[str, Any]:
        """
        Retrieve the entire document from the knowledge base.
        
        Args:
            doc_id: Document ID
            
        Returns:
            Full document content and metadata
        """
        try:
            # First check local document store
            if doc_id in self.document_store:
                return self.document_store[doc_id]
            
            if self.config.type == KnowledgeBaseType.LOCAL:
                return self._get_document_local(doc_id)
            elif self.config.type == KnowledgeBaseType.DIFY:
                return self._get_document_dify(doc_id)
            elif self.config.type == KnowledgeBaseType.RAPTOR:
                return self._get_document_raptor(doc_id)
            elif self.config.type == KnowledgeBaseType.GRAPHRAG:
                return self._get_document_graphrag(doc_id)
            else:
                raise ValueError(f"Unsupported knowledge base type: {self.config.type}")
        except Exception as e:
            logger.error(f"Error retrieving document {doc_id}: {e}")
            return {"error": f"Document {doc_id} not found"}
    
    def _get_document_local(self, doc_id: str) -> Dict[str, Any]:
        """Get document from local retrieval pipeline"""
        try:
            response = requests.get(
                f"{self.config.local_base_url}/documents/{doc_id}"
            )
            
            if response.status_code == 404:
                return {"error": f"Document {doc_id} not found"}
            
            response.raise_for_status()
            return response.json()
            
        except requests.exceptions.RequestException as e:
            logger.error(f"Error getting document from local pipeline: {e}")
            return {"error": str(e)}
    
    def _get_document_dify(self, doc_id: str) -> Dict[str, Any]:
        """Get document from Dify"""
        if not self.config.dify_api_key:
            return {"error": "Dify API key not configured"}
        
        try:
            headers = {
                "Authorization": f"Bearer {self.config.dify_api_key}",
                "Content-Type": "application/json"
            }
            
            response = requests.get(
                f"{self.config.dify_base_url}/documents/{doc_id}",
                headers=headers
            )
            
            if response.status_code == 404:
                return {"error": f"Document {doc_id} not found"}
            
            response.raise_for_status()
            return response.json()
            
        except requests.exceptions.RequestException as e:
            logger.error(f"Error getting document from Dify: {e}")
            return {"error": str(e)}
    
    def _get_document_raptor(self, doc_id: str) -> Dict[str, Any]:
        """Get document/node from RAPTOR index"""
        try:
            # For RAPTOR, we perform a targeted search for the specific node
            response = requests.post(
                f"{self.config.raptor_base_url}/query",
                json={
                    "query": f"node:{doc_id}",  # Specific node query
                    "index_type": "raptor",
                    "top_k": 1
                }
            )
            
            if response.status_code == 404:
                return {"error": f"Document {doc_id} not found"}
            
            response.raise_for_status()
            data = response.json()
            
            if data.get("results"):
                result = data["results"][0]
                return {
                    "doc_id": doc_id,
                    "content": result.get("text", ""),
                    "metadata": {
                        "summary": result.get("summary", ""),
                        "level": result.get("level", 0),
                        "source": "raptor"
                    }
                }
            
            return {"error": f"Document {doc_id} not found"}
            
        except requests.exceptions.RequestException as e:
            logger.error(f"Error getting document from RAPTOR: {e}")
            return {"error": str(e)}
    
    def _get_document_graphrag(self, doc_id: str) -> Dict[str, Any]:
        """Get entity or community from GraphRAG index"""
        try:
            # For GraphRAG, we perform a targeted search for the specific entity/community
            response = requests.post(
                f"{self.config.graphrag_base_url}/query",
                json={
                    "query": f"id:{doc_id}",  # Specific ID query
                    "index_type": "graphrag",
                    "top_k": 1,
                    "search_type": "hybrid"
                }
            )
            
            if response.status_code == 404:
                return {"error": f"Document {doc_id} not found"}
            
            response.raise_for_status()
            data = response.json()
            
            if data.get("results"):
                result = data["results"][0]
                content = ""
                metadata = {"source": "graphrag"}
                
                if result.get("type") == "entity":
                    content = f"{result.get('name', '')}\n\n{result.get('description', '')}"
                    metadata.update({
                        "type": "entity",
                        "entity_type": result.get("entity_type"),
                        "related_entities": result.get("related_entities", [])
                    })
                else:  # community
                    content = result.get("summary", "")
                    metadata.update({
                        "type": "community",
                        "level": result.get("level", 0),
                        "entity_count": result.get("entity_count", 0),
                        "sample_entities": result.get("sample_entities", [])
                    })
                
                return {
                    "doc_id": doc_id,
                    "content": content,
                    "metadata": metadata
                }
            
            return {"error": f"Document {doc_id} not found"}
            
        except requests.exceptions.RequestException as e:
            logger.error(f"Error getting document from GraphRAG: {e}")
            return {"error": str(e)}
    
    def add_document(self, doc_id: str, content: str, metadata: Optional[Dict] = None):
        """Add a document to the local store"""
        self.document_store[doc_id] = {
            "doc_id": doc_id,
            "content": content,
            "metadata": metadata or {}
        }
        self.save_document_store()


# Tool function definitions for agent
def get_tool_definitions() -> List[Dict[str, Any]]:
    """Get OpenAI-format tool definitions"""
    return [
        {
            "type": "function",
            "function": {
                "name": "knowledge_base_search",
                "description": "Search the knowledge base for relevant information using a natural language query. Returns top-matching document chunks.",
                "parameters": {
                    "type": "object",
                    "properties": {
                        "query": {
                            "type": "string",
                            "description": "Natural language search query to find relevant information"
                        }
                    },
                    "required": ["query"]
                }
            }
        },
        {
            "type": "function", 
            "function": {
                "name": "get_document",
                "description": "Retrieve the complete content of a specific document from the knowledge base using its document ID.",
                "parameters": {
                    "type": "object",
                    "properties": {
                        "doc_id": {
                            "type": "string",
                            "description": "The unique identifier of the document to retrieve"
                        }
                    },
                    "required": ["doc_id"]
                }
            }
        }
    ]