◉ClassyGitShadowzzh

txt.py

txt.py
import json
import os
import time
import traceback
from pathlib import Path
from typing import Any, Literal

import chardet
from dotenv import load_dotenv
from pydantic import Field
from pydantic.fields import FieldInfo

from aworld.logs.util import Color
from examples.gaia.mcp_collections.base import ActionArguments, ActionCollection, ActionResponse
from examples.gaia.mcp_collections.documents.models import DocumentMetadata
from examples.gaia.mcp_collections.utils import get_mime_type


class TextExtractionCollection(ActionCollection):
    """MCP service for text document content extraction.

    Supports extraction from TXT and other raw text format files.
    Provides LLM-friendly text output with structured metadata and encoding detection.
    Handles various text encodings and provides comprehensive file analysis.
    """

    def __init__(self, arguments: ActionArguments) -> None:
        super().__init__(arguments)
        self._media_output_dir = self.workspace / "extracted_media"
        self._media_output_dir.mkdir(exist_ok=True)

        self.supported_extensions: set = {
            ".txt",
            ".text",
            ".log",
            ".md",
            ".markdown",
            ".rst",
            ".rtf",
            ".csv",
            ".tsv",
            ".json",
            ".xml",
            ".yaml",
            ".yml",
            ".ini",
            ".cfg",
            ".conf",
            ".properties",
            ".sql",
            ".py",
            ".js",
            ".html",
            ".htm",
            ".css",
            ".java",
            ".cpp",
            ".c",
            ".h",
            ".php",
            ".rb",
            ".go",
            ".rs",
            ".sh",
            ".bat",
            ".ps1",
            ".r",
            ".m",
            ".swift",
            ".kt",
            ".scala",
            ".pl",
            ".lua",
            ".vim",
            ".tex",
            ".bib",
        }

        self._color_log("Text Extraction Service initialized", Color.green, "debug")
        self._color_log(f"Media output directory: {self._media_output_dir}", Color.blue, "debug")

    def _validate_file_path(self, file_path: str) -> Path:
        """Validate and resolve file path.

        Args:
            file_path: Path to the text document file

        Returns:
            Resolved Path object

        Raises:
            FileNotFoundError: If file doesn't exist
            ValueError: If file type is not supported
        """
        path = Path(file_path)
        if not path.is_absolute():
            path = self.workspace / path

        if not path.exists():
            raise FileNotFoundError(f"File not found: {path}")

        # Also check MIME type for files without extensions or unknown extensions
        mime_type = get_mime_type(str(path), default_mime="text/plain")
        is_text_mime = mime_type and mime_type.startswith("text/")

        if path.suffix.lower() not in self.supported_extensions and not is_text_mime:
            # Try to detect if it's a text file by reading a small sample
            try:
                with open(path, "rb") as f:
                    sample = f.read(1024)
                    # Check if the sample contains mostly printable characters
                    if self._is_likely_text(sample):
                        self._color_log(f"Detected text file without standard extension: {path.suffix}", Color.yellow)
                    else:
                        raise ValueError(
                            f"Unsupported file type: {path.suffix}. "
                            f"Supported types: {', '.join(sorted(self.supported_extensions))} or text MIME types"
                        )
            except Exception as e:
                raise ValueError(
                    f"Cannot determine if file is text: {str(e)}. "
                    f"Supported types: {', '.join(sorted(self.supported_extensions))}"
                ) from e

        return path

    def _is_likely_text(self, data: bytes) -> bool:
        """Check if binary data is likely to be text.

        Args:
            data: Binary data sample

        Returns:
            True if data appears to be text
        """
        if not data:
            return True

        # Check for null bytes (common in binary files)
        if b"\x00" in data:
            return False

        # Try to decode as text
        try:
            data.decode("utf-8")
            return True
        except UnicodeDecodeError:
            pass

        # Check if most bytes are printable ASCII
        printable_count = sum(1 for byte in data if 32 <= byte <= 126 or byte in [9, 10, 13])
        return printable_count / len(data) > 0.7

    def _detect_encoding(self, file_path: Path) -> dict[str, Any]:
        """Detect file encoding and other characteristics.

        Args:
            file_path: Path to the text file

        Returns:
            Dictionary containing encoding information
        """
        encoding_info = {
            "detected_encoding": None,
            "confidence": 0.0,
            "bom_detected": False,
            "line_endings": None,
            "is_binary": False,
        }

        try:
            # Read file in binary mode for encoding detection
            with open(file_path, "rb") as f:
                raw_data = f.read()

            if not raw_data:
                encoding_info["detected_encoding"] = "utf-8"
                encoding_info["confidence"] = 1.0
                return encoding_info

            # Check for BOM (Byte Order Mark)
            if raw_data.startswith(b"\xef\xbb\xbf"):
                encoding_info["bom_detected"] = True
                encoding_info["detected_encoding"] = "utf-8-sig"
                encoding_info["confidence"] = 1.0
            elif raw_data.startswith(b"\xff\xfe"):
                encoding_info["bom_detected"] = True
                encoding_info["detected_encoding"] = "utf-16-le"
                encoding_info["confidence"] = 1.0
            elif raw_data.startswith(b"\xfe\xff"):
                encoding_info["bom_detected"] = True
                encoding_info["detected_encoding"] = "utf-16-be"
                encoding_info["confidence"] = 1.0
            else:
                # Use chardet for encoding detection
                detection_result = chardet.detect(raw_data)
                encoding_info["detected_encoding"] = detection_result.get("encoding", "utf-8")
                encoding_info["confidence"] = detection_result.get("confidence", 0.0)

            # Detect line endings
            if b"\r\n" in raw_data:
                encoding_info["line_endings"] = "CRLF (Windows)"
            elif b"\n" in raw_data:
                encoding_info["line_endings"] = "LF (Unix/Linux/Mac)"
            elif b"\r" in raw_data:
                encoding_info["line_endings"] = "CR (Classic Mac)"
            else:
                encoding_info["line_endings"] = "None detected"

            # Check if file appears to be binary
            encoding_info["is_binary"] = not self._is_likely_text(raw_data[:1024])

        except Exception as e:
            self.logger.warning(f"Failed to detect encoding: {str(e)}")
            encoding_info["detected_encoding"] = "utf-8"
            encoding_info["confidence"] = 0.0

        return encoding_info

    def _extract_text_content(self, file_path: Path, encoding: str | None = None) -> dict[str, Any]:
        """Extract content from text files.

        Args:
            file_path: Path to the text file
            encoding: Specific encoding to use (None for auto-detection)

        Returns:
            Dictionary containing extracted content and metadata
        """
        start_time = time.time()

        # Detect encoding if not specified
        encoding_info = self._detect_encoding(file_path)

        if encoding:
            # Use specified encoding
            target_encoding = encoding
            self._color_log(f"Using specified encoding: {encoding}", Color.blue)
        else:
            # Use detected encoding
            target_encoding = encoding_info["detected_encoding"]
            self._color_log(
                f"Detected encoding: {target_encoding} (confidence: {encoding_info['confidence']:.2f})", Color.blue
            )

        try:
            # Read file content
            with open(file_path, "r", encoding=target_encoding, errors="replace") as f:
                content = f.read()

            # Analyze content
            lines = content.splitlines()

            # Calculate statistics
            char_count = len(content)
            line_count = len(lines)
            word_count = len(content.split()) if content.strip() else 0

            # Find longest and shortest lines
            line_lengths = [len(line) for line in lines]
            max_line_length = max(line_lengths) if line_lengths else 0
            min_line_length = min(line_lengths) if line_lengths else 0
            avg_line_length = sum(line_lengths) / len(line_lengths) if line_lengths else 0

            # Count empty lines
            empty_lines = sum(1 for line in lines if not line.strip())

            # Detect file type based on content patterns
            content_type = self._detect_content_type(content, file_path)

            processing_time = time.time() - start_time

            return {
                "content": content,
                "encoding_info": encoding_info,
                "statistics": {
                    "character_count": char_count,
                    "line_count": line_count,
                    "word_count": word_count,
                    "empty_lines": empty_lines,
                    "max_line_length": max_line_length,
                    "min_line_length": min_line_length,
                    "avg_line_length": round(avg_line_length, 2),
                },
                "content_type": content_type,
                "processing_time": processing_time,
                "used_encoding": target_encoding,
            }

        except UnicodeDecodeError as e:
            self.logger.error(f"Failed to decode file with encoding {target_encoding}: {str(e)}")
            # Try with fallback encodings
            fallback_encodings = ["utf-8", "latin-1", "cp1252", "iso-8859-1"]

            for fallback_encoding in fallback_encodings:
                if fallback_encoding != target_encoding:
                    try:
                        with open(file_path, "r", encoding=fallback_encoding, errors="replace") as f:
                            content = f.read()

                        self._color_log(f"Successfully read with fallback encoding: {fallback_encoding}", Color.yellow)

                        # Recalculate with fallback encoding
                        lines = content.splitlines()
                        char_count = len(content)
                        line_count = len(lines)
                        word_count = len(content.split()) if content.strip() else 0

                        processing_time = time.time() - start_time

                        return {
                            "content": content,
                            "encoding_info": encoding_info,
                            "statistics": {
                                "character_count": char_count,
                                "line_count": line_count,
                                "word_count": word_count,
                                "empty_lines": sum(1 for line in lines if not line.strip()),
                                "max_line_length": max(len(line) for line in lines) if lines else 0,
                                "min_line_length": min(len(line) for line in lines) if lines else 0,
                                "avg_line_length": round(sum(len(line) for line in lines) / len(lines), 2)
                                if lines
                                else 0,
                            },
                            "content_type": self._detect_content_type(content, file_path),
                            "processing_time": processing_time,
                            "used_encoding": fallback_encoding,
                            "encoding_fallback": True,
                        }
                    except Exception:
                        continue

            raise ValueError("Unable to decode file with any supported encoding") from e

    def _detect_content_type(self, content: str, file_path: Path) -> str:
        """Detect the type of content based on file extension and content patterns.

        Args:
            content: File content
            file_path: Path to the file

        Returns:
            Detected content type
        """
        extension = file_path.suffix.lower()

        # Map extensions to content types
        extension_map = {
            ".py": "Python source code",
            ".js": "JavaScript source code",
            ".html": "HTML document",
            ".htm": "HTML document",
            ".css": "CSS stylesheet",
            ".json": "JSON data",
            ".xml": "XML document",
            ".yaml": "YAML configuration",
            ".yml": "YAML configuration",
            ".md": "Markdown document",
            ".markdown": "Markdown document",
            ".rst": "reStructuredText document",
            ".csv": "CSV data",
            ".tsv": "TSV data",
            ".sql": "SQL script",
            ".log": "Log file",
            ".ini": "Configuration file",
            ".cfg": "Configuration file",
            ".conf": "Configuration file",
        }

        if extension in extension_map:
            return extension_map[extension]

        # Content-based detection
        content_lower = content.lower().strip()

        if content_lower.startswith("<?xml"):
            return "XML document"
        elif content_lower.startswith("{") and content_lower.endswith("}"):
            return "JSON-like data"
        elif content_lower.startswith("[") and content_lower.endswith("]"):
            return "JSON array or configuration"
        elif "#!/" in content[:50]:
            return "Script file"
        elif content.count(",") > content.count("\n") * 2:
            return "CSV-like data"
        else:
            return "Plain text"

    def _format_content_for_llm(
        self, extraction_result: dict[str, Any], output_format: str, max_length: int | None = None
    ) -> str:
        """Format extracted text content to be LLM-friendly.

        Args:
            extraction_result: Result from _extract_text_content
            output_format: Desired output format
            max_length: Maximum length of content to include (None for no limit)

        Returns:
            Formatted content string
        """
        content = extraction_result["content"]
        stats = extraction_result["statistics"]
        content_type = extraction_result["content_type"]

        # Truncate content if needed
        if max_length and len(content) > max_length:
            content = (
                content[:max_length]
                + f"\n\n[Content truncated - showing first {max_length} characters of {stats['character_count']} total]"
            )

        if output_format.lower() == "markdown":
            formatted_parts = []
            formatted_parts.append("# Text Document Content\n")
            formatted_parts.append(f"**File Type:** {content_type}\n")
            formatted_parts.append(f"**Encoding:** {extraction_result['used_encoding']}\n")
            formatted_parts.append("**Statistics:**\n")
            formatted_parts.append(f"- Characters: {stats['character_count']:,}\n")
            formatted_parts.append(f"- Lines: {stats['line_count']:,}\n")
            formatted_parts.append(f"- Words: {stats['word_count']:,}\n")
            formatted_parts.append(f"- Empty lines: {stats['empty_lines']:,}\n")
            formatted_parts.append(f"- Average line length: {stats['avg_line_length']} characters\n\n")

            formatted_parts.append(f"## Content\n\n```\n{content}\n```")

            return "".join(formatted_parts)

        elif output_format.lower() == "json":
            json_data = {
                "document_info": {
                    "content_type": content_type,
                    "encoding": extraction_result["used_encoding"],
                    "statistics": stats,
                },
                "content": content,
            }

            return json.dumps(json_data, indent=2, ensure_ascii=False)

        elif output_format.lower() == "html":
            html_parts = []
            html_parts.append("<html><head><meta charset='utf-8'></head><body>")
            html_parts.append("<h1>Text Document Content</h1>")
            html_parts.append(f"<p><strong>File Type:</strong> {content_type}</p>")
            html_parts.append(f"<p><strong>Encoding:</strong> {extraction_result['used_encoding']}</p>")
            html_parts.append("<h2>Statistics</h2>")
            html_parts.append("<ul>")
            html_parts.append(f"<li>Characters: {stats['character_count']:,}</li>")
            html_parts.append(f"<li>Lines: {stats['line_count']:,}</li>")
            html_parts.append(f"<li>Words: {stats['word_count']:,}</li>")
            html_parts.append(f"<li>Empty lines: {stats['empty_lines']:,}</li>")
            html_parts.append(f"<li>Average line length: {stats['avg_line_length']} characters</li>")
            html_parts.append("</ul>")
            html_parts.append("<h2>Content</h2>")
            html_parts.append(f"<pre><code>{content}</code></pre>")
            html_parts.append("</body></html>")

            return "".join(html_parts)

        else:  # text format
            text_parts = []
            text_parts.append(f"Text Document Content\n{'=' * 50}\n")
            text_parts.append(f"File Type: {content_type}\n")
            text_parts.append(f"Encoding: {extraction_result['used_encoding']}\n")
            text_parts.append("\nStatistics:\n")
            text_parts.append(f"  Characters: {stats['character_count']:,}\n")
            text_parts.append(f"  Lines: {stats['line_count']:,}\n")
            text_parts.append(f"  Words: {stats['word_count']:,}\n")
            text_parts.append(f"  Empty lines: {stats['empty_lines']:,}\n")
            text_parts.append(f"  Average line length: {stats['avg_line_length']} characters\n")
            text_parts.append(f"\nContent:\n{'-' * 30}\n{content}")

            return "".join(text_parts)

    def mcp_extract_text_content(
        self,
        file_path: str = Field(description="Path to the text document file to extract content from"),
        output_format: Literal["markdown", "json", "html", "text"] = Field(
            default="markdown", description="Output format: 'markdown', 'json', 'html', or 'text'"
        ),
        encoding: str | None = Field(default=None, description="Specific encoding to use (None for auto-detection)"),
        max_content_length: int | None = Field(
            default=None, description="Maximum length of content to include in output (None for no limit)"
        ),
    ) -> ActionResponse:
        """Extract content from text documents with encoding detection and analysis.

        This tool provides comprehensive text document content extraction with support for:
        - Various text file formats (TXT, MD, CSV, JSON, XML, source code, etc.)
        - Automatic encoding detection with fallback options
        - Content type detection and analysis
        - Comprehensive text statistics
        - LLM-optimized output formatting
        - Binary file detection and handling

        Args:
            file_path: Path to the text file
            output_format: Desired output format
            encoding: Specific encoding to use
            max_content_length: Maximum content length to include

        Returns:
            ActionResponse with extracted content, metadata, and file analysis
        """
        try:
            # Handle FieldInfo objects
            if isinstance(file_path, FieldInfo):
                file_path = file_path.default
            if isinstance(output_format, FieldInfo):
                output_format = output_format.default
            if isinstance(encoding, FieldInfo):
                encoding = encoding.default
            if isinstance(max_content_length, FieldInfo):
                max_content_length = max_content_length.default

            # Validate input file
            file_path: Path = self._validate_file_path(file_path)
            self._color_log(f"Processing text document: {file_path.name}", Color.cyan)

            # Extract content from text file
            extraction_result = self._extract_text_content(file_path, encoding)

            # Check if file appears to be binary
            if extraction_result["encoding_info"]["is_binary"]:
                self._color_log("Warning: File appears to contain binary data", Color.yellow)

            # Format content for LLM consumption
            formatted_content = self._format_content_for_llm(extraction_result, output_format, max_content_length)

            # Prepare metadata
            file_stats = file_path.stat()

            # Create text-specific metadata
            text_metadata = {
                "content_type": extraction_result["content_type"],
                "encoding_info": extraction_result["encoding_info"],
                "text_statistics": extraction_result["statistics"],
                "used_encoding": extraction_result["used_encoding"],
                "encoding_fallback": extraction_result.get("encoding_fallback", False),
                "content_truncated": max_content_length and len(extraction_result["content"]) > max_content_length,
                "original_content_length": extraction_result["statistics"]["character_count"],
            }

            document_metadata = DocumentMetadata(
                file_name=file_path.name,
                file_size=file_stats.st_size,
                file_type=file_path.suffix.lower() or ".txt",
                absolute_path=str(file_path.absolute()),
                page_count=extraction_result["statistics"]["line_count"],  # Use line count as "page" count
                processing_time=extraction_result["processing_time"],
                extracted_images=[],  # Text files don't contain images
                extracted_media=[],  # Text files don't contain media
                output_format=output_format,
                llm_enhanced=False,
                ocr_applied=False,
            )

            # Combine standard and text-specific metadata
            combined_metadata = document_metadata.model_dump()
            combined_metadata.update(text_metadata)

            self._color_log(
                f"Successfully extracted content from {file_path.name} "
                f"({extraction_result['statistics']['character_count']:,} characters, "
                f"{extraction_result['statistics']['line_count']:,} lines, "
                f"encoding: {extraction_result['used_encoding']})",
                Color.green,
            )

            return ActionResponse(success=True, message=formatted_content, metadata=combined_metadata)

        except FileNotFoundError as e:
            self.logger.error(f"File not found: {str(e)}: {traceback.format_exc()}")
            return ActionResponse(
                success=False, message=f"File not found: {str(e)}", metadata={"error_type": "file_not_found"}
            )
        except ValueError as e:
            self.logger.error(f"Invalid input: {str(e)}: {traceback.format_exc()}")
            return ActionResponse(
                success=False,
                message=f"Invalid input: {str(e)}",
                metadata={"error_type": "invalid_input"},
            )
        except Exception as e:
            self.logger.error(f"Text extraction failed: {str(e)}: {traceback.format_exc()}")
            return ActionResponse(
                success=False,
                message=f"Text extraction failed: {str(e)}",
                metadata={"error_type": "extraction_error"},
            )

    def mcp_list_supported_formats(self) -> ActionResponse:
        """List all supported text formats for extraction.

        Returns:
            ActionResponse with list of supported file formats and their descriptions
        """
        supported_formats = {
            "TXT": "Plain text files (.txt, .text)",
            "Markdown": "Markdown documents (.md, .markdown)",
            "CSV/TSV": "Comma/Tab separated values (.csv, .tsv)",
            "JSON": "JSON data files (.json)",
            "XML": "XML documents (.xml)",
            "YAML": "YAML configuration files (.yaml, .yml)",
            "Source Code": "Programming language files (.py, .js, .html, .css, etc.)",
            "Configuration": "Config files (.ini, .cfg, .conf, .properties)",
            "Logs": "Log files (.log)",
            "Documentation": "Documentation files (.rst, .rtf)",
            "Scripts": "Script files (.sh, .bat, .ps1)",
            "Other Text": "Any file with text MIME type or detectable text content",
        }

        format_list = "\n".join(
            [f"**{format_name}**: {description}" for format_name, description in supported_formats.items()]
        )

        return ActionResponse(
            success=True,
            message=f"Supported text formats:\n\n{format_list}\n\n"
            "**Note:** The service automatically detects encoding and "
            "can handle files without standard extensions if they contain readable text.",
            metadata={
                "supported_formats": list(supported_formats.keys()),
                "total_formats": len(supported_formats),
                "encoding_detection": True,
                "binary_detection": True,
            },
        )


# Example usage and entry point
if __name__ == "__main__":
    load_dotenv()

    # Default arguments for testing
    args = ActionArguments(
        name="text_extraction_service",
        transport="stdio",
        workspace=os.getenv("AWORLD_WORKSPACE", "~"),
    )

    # Initialize and run the text extraction service
    try:
        service = TextExtractionCollection(args)
        service.run()
    except Exception as e:
        print(f"An error occurred: {e}: {traceback.format_exc()}")