ClassyGitShadowzzh

media_processing_tools.py

media_processing_tools.py
"""
Media processing tools for audio, image, and video.
Based on AWorld MCP server implementation.
"""
import json
import logging
import traceback
import subprocess
import base64
from pathlib import Path
from typing import Union, Dict, Any

import cv2
from PIL import Image
from dotenv import load_dotenv
from mcp.types import TextContent

from base import ActionResponse, validate_file_path


load_dotenv()


def _map_model_for_openrouter(model: str) -> str:
    """Map a plain model id onto OpenRouter's `provider/model` form."""
    if "/" in model:
        return model
    m = model.lower()
    if m.startswith(("gpt-", "o1-", "o3-", "o4-")):
        return f"openai/{model}"
    if m.startswith("claude-"):
        return "anthropic/claude-opus-4.8"
    return model


def _make_vision_client(default_model: str = "gpt-5.6-luna"):
    """Build an OpenAI-compatible vision client with a universal fallback.

    Preferred path uses OPENAI_API_KEY directly. When it is absent but an
    OPENROUTER_API_KEY is set, transparently route through OpenRouter (mapping
    the model id to provider/model form) so the vision tools still run.

    Returns (client, model). Raises ValueError with the accepted keys listed
    when neither credential is available.
    """
    import os
    from openai import OpenAI

    model = os.getenv("PERCEPTION_VISION_MODEL", default_model)
    or_key = os.getenv("OPENROUTER_API_KEY")
    # gpt-5.x (incl. gpt-5.6*) needs OpenAI org-verification on the direct API;
    # when an OpenRouter key is present, prefer routing these ids through it.
    if or_key and model.lower().startswith("gpt-5"):
        client = OpenAI(api_key=or_key, base_url="https://openrouter.ai/api/v1")
        return client, _map_model_for_openrouter(model)

    api_key = os.getenv("OPENAI_API_KEY")
    if api_key:
        base_url = os.getenv("OPENAI_BASE_URL")
        client = OpenAI(api_key=api_key, base_url=base_url) if base_url else OpenAI(api_key=api_key)
        return client, model

    if or_key:
        client = OpenAI(api_key=or_key, base_url="https://openrouter.ai/api/v1")
        return client, _map_model_for_openrouter(model)

    raise ValueError(
        "No LLM key configured. Set OPENAI_API_KEY or OPENROUTER_API_KEY (universal fallback)."
    )


async def transcribe_audio_whisper(
    file_path: str,
    model_size: str = "base",
    language: str = "en"
) -> Union[str, TextContent]:
    """
    Transcribe audio file using OpenAI Whisper (local).
    Note: Requires whisper package installed.
    
    Args:
        file_path: Path to audio file
        model_size: Whisper model size (tiny, base, small, medium, large)
        language: Language code
        
    Returns:
        TextContent with transcription
    """
    try:
        path = validate_file_path(file_path)
        
        logging.info(f"🎤 Transcribing audio: {path}")
        
        try:
            import whisper
            
            # Load model
            model = whisper.load_model(model_size)
            
            # Transcribe
            result = model.transcribe(str(path), language=language)
            
            transcription = result["text"]
            
            response_data = {
                "file_name": path.name,
                "file_type": path.suffix,
                "model": model_size,
                "language": language,
                "transcription": transcription,
                "word_count": len(transcription.split())
            }
            
            logging.info(f"✅ Transcribed: {len(transcription)} chars")
            
            action_response = ActionResponse(
                success=True,
                message=response_data,
                metadata={"file_path": str(path)}
            )
            
        except ImportError:
            # Fallback: try using OpenAI API if available
            import os
            from openai import OpenAI
            
            api_key = os.getenv("OPENAI_API_KEY")
            if not api_key:
                raise ImportError("Whisper not installed and no OPENAI_API_KEY found")
            
            client = OpenAI(api_key=api_key)
            
            with open(path, "rb") as audio_file:
                transcription = client.audio.transcriptions.create(
                    model="whisper-1",
                    file=audio_file,
                    language=language
                )
            
            response_data = {
                "file_name": path.name,
                "file_type": path.suffix,
                "model": "whisper-1 (API)",
                "language": language,
                "transcription": transcription.text,
                "word_count": len(transcription.text.split())
            }
            
            action_response = ActionResponse(
                success=True,
                message=response_data,
                metadata={"file_path": str(path), "method": "openai_api"}
            )
        
        return TextContent(
            type="text",
            text=json.dumps(action_response.model_dump())
        )
        
    except Exception as e:
        error_msg = f"Audio transcription failed: {str(e)}"
        logging.error(f"Audio error: {traceback.format_exc()}")
        
        action_response = ActionResponse(
            success=False,
            message=error_msg,
            metadata={"error_type": "audio_error"}
        )
        
        return TextContent(
            type="text",
            text=json.dumps(action_response.model_dump())
        )


async def extract_audio_metadata(
    file_path: str
) -> Union[str, TextContent]:
    """
    Extract audio file metadata using ffprobe.
    
    Args:
        file_path: Path to audio file
        
    Returns:
        TextContent with audio metadata
    """
    try:
        path = validate_file_path(file_path)
        
        logging.info(f"🎵 Extracting audio metadata: {path}")
        
        # Use ffprobe to get metadata
        cmd = [
            "ffprobe",
            "-v", "quiet",
            "-print_format", "json",
            "-show_format",
            "-show_streams",
            str(path)
        ]
        
        result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
        
        if result.returncode == 0:
            metadata = json.loads(result.stdout)
            
            format_info = metadata.get("format", {})
            streams = metadata.get("streams", [])
            audio_stream = next((s for s in streams if s.get("codec_type") == "audio"), {})
            
            response_data = {
                "file_name": path.name,
                "file_size": path.stat().st_size,
                "duration": float(format_info.get("duration", 0)),
                "bit_rate": int(format_info.get("bit_rate", 0)),
                "format": format_info.get("format_name"),
                "codec": audio_stream.get("codec_name"),
                "sample_rate": int(audio_stream.get("sample_rate", 0)) if audio_stream.get("sample_rate") else None,
                "channels": int(audio_stream.get("channels", 0)) if audio_stream.get("channels") else None
            }
            
            logging.info(f"✅ Audio metadata extracted")
            
            action_response = ActionResponse(
                success=True,
                message=response_data,
                metadata={"file_path": str(path)}
            )
        else:
            raise RuntimeError(f"ffprobe failed: {result.stderr}")
        
        return TextContent(
            type="text",
            text=json.dumps(action_response.model_dump())
        )
        
    except Exception as e:
        error_msg = f"Audio metadata extraction failed: {str(e)}"
        logging.error(f"Audio metadata error: {traceback.format_exc()}")
        
        action_response = ActionResponse(
            success=False,
            message=error_msg,
            metadata={"error_type": "audio_metadata_error"}
        )
        
        return TextContent(
            type="text",
            text=json.dumps(action_response.model_dump())
        )


async def extract_text_ocr(
    image_path: str,
    language: str = "eng"
) -> Union[str, TextContent]:
    """
    Extract text from image using OCR.
    
    Args:
        image_path: Path to image file
        language: OCR language (eng, chi_sim, etc.)
        
    Returns:
        TextContent with extracted text
    """
    try:
        path = validate_file_path(image_path)
        
        logging.info(f"🔍 OCR extracting from image: {path}")
        
        try:
            import pytesseract
            
            img = Image.open(path)
            text = pytesseract.image_to_string(img, lang=language)
            
            result = {
                "file_name": path.name,
                "image_size": img.size,
                "extracted_text": text,
                "text_length": len(text),
                "word_count": len(text.split()),
                "language": language,
                "method": "pytesseract"
            }
            
            logging.info(f"✅ OCR extracted: {len(text)} chars")
            
            action_response = ActionResponse(
                success=True,
                message=result,
                metadata={"file_path": str(path)}
            )
            
        except ImportError:
            # Fallback to a simpler method or error
            raise ImportError("pytesseract not installed. Install with: pip install pytesseract")
        
        return TextContent(
            type="text",
            text=json.dumps(action_response.model_dump())
        )
        
    except Exception as e:
        error_msg = f"OCR extraction failed: {str(e)}"
        logging.error(f"OCR error: {traceback.format_exc()}")
        
        action_response = ActionResponse(
            success=False,
            message=error_msg,
            metadata={"error_type": "ocr_error"}
        )
        
        return TextContent(
            type="text",
            text=json.dumps(action_response.model_dump())
        )


async def analyze_image_ai(
    image_path: str,
    prompt: str = "Describe this image in detail"
) -> Union[str, TextContent]:
    """
    Analyze image using AI (OpenAI Vision API).
    
    Args:
        image_path: Path to image file
        prompt: Prompt for AI analysis
        
    Returns:
        TextContent with AI analysis
    """
    try:
        path = validate_file_path(image_path)

        logging.info(f"🤖 AI analyzing image: {path}")

        client, model = _make_vision_client()

        # Encode image
        with open(path, "rb") as img_file:
            img_base64 = base64.b64encode(img_file.read()).decode('utf-8')

        # Call Vision API
        response = client.chat.completions.create(
            model=model,
            messages=[
                {
                    "role": "user",
                    "content": [
                        {"type": "text", "text": prompt},
                        {
                            "type": "image_url",
                            "image_url": {
                                "url": f"data:image/jpeg;base64,{img_base64}"
                            }
                        }
                    ]
                }
            ],
            max_tokens=500
        )

        analysis = response.choices[0].message.content

        result = {
            "file_name": path.name,
            "prompt": prompt,
            "analysis": analysis,
            "model": model
        }
        
        logging.info(f"✅ AI analysis completed")
        
        action_response = ActionResponse(
            success=True,
            message=result,
            metadata={"file_path": str(path)}
        )
        
        return TextContent(
            type="text",
            text=json.dumps(action_response.model_dump())
        )
        
    except Exception as e:
        error_msg = f"AI image analysis failed: {str(e)}"
        logging.error(f"AI analysis error: {traceback.format_exc()}")
        
        action_response = ActionResponse(
            success=False,
            message=error_msg,
            metadata={"error_type": "ai_analysis_error"}
        )
        
        return TextContent(
            type="text",
            text=json.dumps(action_response.model_dump())
        )


async def extract_video_keyframes(
    video_path: str,
    num_frames: int = 10
) -> Union[str, TextContent]:
    """
    Extract keyframes from video.
    
    Args:
        video_path: Path to video file
        num_frames: Number of keyframes to extract
        
    Returns:
        TextContent with keyframe information
    """
    try:
        path = validate_file_path(video_path)
        
        num_frames = max(1, num_frames)
        
        logging.info(f"🎬 Extracting keyframes from video: {path}")
        
        video = cv2.VideoCapture(str(path))
        
        fps = video.get(cv2.CAP_PROP_FPS)
        frame_count = int(video.get(cv2.CAP_PROP_FRAME_COUNT))
        duration = frame_count / fps if fps > 0 else 0
        
        # Calculate frame interval
        interval = max(1, frame_count // num_frames)
        
        keyframes = []
        frame_num = 0
        
        while len(keyframes) < num_frames and video.isOpened():
            ret, frame = video.read()
            if not ret:
                break
            
            if frame_num % interval == 0:
                timestamp = frame_num / fps if fps > 0 else 0
                keyframes.append({
                    "frame_number": frame_num,
                    "timestamp": round(timestamp, 2),
                    "shape": frame.shape if frame is not None else None
                })
            
            frame_num += 1
        
        video.release()
        
        result = {
            "file_name": path.name,
            "duration": duration,
            "total_frames": frame_count,
            "fps": fps,
            "keyframes_extracted": len(keyframes),
            "keyframes": keyframes
        }
        
        logging.info(f"✅ Extracted {len(keyframes)} keyframes")
        
        action_response = ActionResponse(
            success=True,
            message=result,
            metadata={"file_path": str(path)}
        )
        
        return TextContent(
            type="text",
            text=json.dumps(action_response.model_dump())
        )
        
    except Exception as e:
        error_msg = f"Keyframe extraction failed: {str(e)}"
        logging.error(f"Video error: {traceback.format_exc()}")
        
        action_response = ActionResponse(
            success=False,
            message=error_msg,
            metadata={"error_type": "video_error"}
        )
        
        return TextContent(
            type="text",
            text=json.dumps(action_response.model_dump())
        )


async def analyze_video_ai(
    video_path: str,
    num_frames: int = 5,
    prompt: str = "Analyze this video and describe what's happening"
) -> Union[str, TextContent]:
    """
    Analyze video content using AI vision on keyframes.
    
    Args:
        video_path: Path to video file
        num_frames: Number of frames to analyze
        prompt: Analysis prompt for AI
        
    Returns:
        TextContent with AI analysis
    """
    try:
        path = validate_file_path(video_path)

        num_frames = max(1, num_frames)

        logging.info(f"🤖 AI analyzing video: {path}")

        client, model = _make_vision_client()

        # Extract keyframes
        video = cv2.VideoCapture(str(path))
        fps = video.get(cv2.CAP_PROP_FPS)
        frame_count = int(video.get(cv2.CAP_PROP_FRAME_COUNT))
        interval = max(1, frame_count // num_frames)
        
        # Extract and encode frames
        frame_analyses = []
        frame_num = 0
        frames_analyzed = 0
        
        while frames_analyzed < num_frames and video.isOpened():
            ret, frame = video.read()
            if not ret:
                break
            
            if frame_num % interval == 0:
                # Encode frame
                _, buffer = cv2.imencode('.jpg', frame)
                img_base64 = base64.b64encode(buffer).decode('utf-8')
                
                # Analyze with GPT-4 Vision
                response = client.chat.completions.create(
                    model=model,
                    messages=[
                        {
                            "role": "user",
                            "content": [
                                {"type": "text", "text": f"{prompt} (Frame {frames_analyzed + 1}/{num_frames})"},
                                {
                                    "type": "image_url",
                                    "image_url": {"url": f"data:image/jpeg;base64,{img_base64}"}
                                }
                            ]
                        }
                    ],
                    max_tokens=300
                )
                
                timestamp = frame_num / fps if fps > 0 else 0
                analysis = response.choices[0].message.content
                
                frame_analyses.append({
                    "frame_number": frame_num,
                    "timestamp": round(timestamp, 2),
                    "analysis": analysis
                })
                
                frames_analyzed += 1
            
            frame_num += 1
        
        video.release()
        
        # Generate overall summary
        combined_analyses = "\n\n".join([f"Frame {i+1} (t={a['timestamp']}s): {a['analysis']}" 
                                          for i, a in enumerate(frame_analyses)])
        
        result = {
            "file_name": path.name,
            "frames_analyzed": len(frame_analyses),
            "analyses": frame_analyses,
            "combined_analysis": combined_analyses
        }
        
        logging.info(f"✅ Analyzed {len(frame_analyses)} frames")
        
        action_response = ActionResponse(
            success=True,
            message=result,
            metadata={"file_path": str(path)}
        )
        
        return TextContent(
            type="text",
            text=json.dumps(action_response.model_dump())
        )
        
    except Exception as e:
        error_msg = f"Video analysis failed: {str(e)}"
        logging.error(f"Video analysis error: {traceback.format_exc()}")
        
        action_response = ActionResponse(
            success=False,
            message=error_msg,
            metadata={"error_type": "video_analysis_error"}
        )
        
        return TextContent(
            type="text",
            text=json.dumps(action_response.model_dump())
        )


async def trim_audio(
    audio_path: str,
    start_time: float,
    duration: float | None = None,
    output_path: str | None = None
) -> Union[str, TextContent]:
    """
    Trim audio file to specified time range using ffmpeg.
    
    Args:
        audio_path: Path to audio file
        start_time: Start time in seconds
        duration: Duration in seconds (None for trim to end)
        output_path: Output file path (None for auto-generate)
        
    Returns:
        TextContent with trimmed audio info
    """
    try:
        path = validate_file_path(audio_path)
        
        logging.info(f"✂️ Trimming audio: {path}")
        
        # Generate output path if not provided
        if output_path is None:
            output_path = str(path.parent / f"{path.stem}_trimmed{path.suffix}")
        
        # Build ffmpeg command
        cmd = ["ffmpeg", "-i", str(path), "-ss", str(start_time)]
        
        if duration is not None:
            cmd.extend(["-t", str(duration)])
        
        cmd.extend(["-c", "copy", "-y", output_path])
        
        # Execute ffmpeg
        result = subprocess.run(cmd, capture_output=True, text=True, timeout=300)
        
        if result.returncode != 0:
            raise RuntimeError(f"ffmpeg failed: {result.stderr}")
        
        output_file = Path(output_path)
        
        response_data = {
            "input_file": str(path),
            "output_file": str(output_file),
            "start_time": start_time,
            "duration": duration,
            "file_size": output_file.stat().st_size if output_file.exists() else 0
        }
        
        logging.info(f"✅ Trimmed audio saved to: {output_file}")
        
        action_response = ActionResponse(
            success=True,
            message=response_data,
            metadata={"output_path": str(output_file)}
        )
        
        return TextContent(
            type="text",
            text=json.dumps(action_response.model_dump())
        )
        
    except Exception as e:
        error_msg = f"Audio trim failed: {str(e)}"
        logging.error(f"Audio trim error: {traceback.format_exc()}")
        
        action_response = ActionResponse(
            success=False,
            message=error_msg,
            metadata={"error_type": "audio_trim_error"}
        )
        
        return TextContent(
            type="text",
            text=json.dumps(action_response.model_dump())
        )


async def get_image_metadata(
    image_path: str
) -> Union[str, TextContent]:
    """
    Get detailed image metadata including EXIF data.
    
    Args:
        image_path: Path to image file
        
    Returns:
        TextContent with image metadata
    """
    try:
        path = validate_file_path(image_path)
        
        logging.info(f"📷 Getting image metadata: {path}")
        
        img = Image.open(path)
        
        # Basic metadata
        metadata = {
            "file_name": path.name,
            "format": img.format,
            "mode": img.mode,
            "size": img.size,
            "width": img.width,
            "height": img.height,
            "file_size": path.stat().st_size
        }
        
        # Try to get EXIF data
        try:
            from PIL.ExifTags import TAGS
            exif_data = {}
            
            if hasattr(img, '_getexif') and img._getexif():
                exif = img._getexif()
                for tag_id, value in exif.items():
                    tag = TAGS.get(tag_id, tag_id)
                    exif_data[tag] = str(value)
            
            if exif_data:
                metadata["exif"] = exif_data
        except Exception as e:
            logging.debug(f"No EXIF data: {e}")
        
        # Image info
        if hasattr(img, 'info'):
            metadata["info"] = dict(img.info)
        
        result = {
            "metadata": metadata,
            "has_exif": "exif" in metadata
        }
        
        logging.info(f"✅ Image metadata extracted")
        
        action_response = ActionResponse(
            success=True,
            message=result,
            metadata={"file_path": str(path)}
        )
        
        return TextContent(
            type="text",
            text=json.dumps(action_response.model_dump())
        )
        
    except Exception as e:
        error_msg = f"Metadata extraction failed: {str(e)}"
        logging.error(f"Metadata error: {traceback.format_exc()}")
        
        action_response = ActionResponse(
            success=False,
            message=error_msg,
            metadata={"error_type": "metadata_error"}
        )
        
        return TextContent(
            type="text",
            text=json.dumps(action_response.model_dump())
        )