ClassyGitshadowzzh

config.py

config.py
"""Configuration module for User Memory Evaluation Framework."""

import os
from typing import Optional
from dotenv import load_dotenv

# Load environment variables
load_dotenv()


def _openrouter_model_id(model) -> str:
    """Map a provider-native model name to an OpenRouter model id, used by the
    universal OpenRouter fallback. An explicit OPENROUTER_MODEL env var wins."""
    override = os.getenv("OPENROUTER_MODEL")
    if override:
        return override
    m = (model or "").strip()
    if not m:
        return "openai/gpt-5.6-luna"
    if "/" in m:
        return m
    ml = m.lower()
    if ml.startswith(("gpt-", "o1", "o3", "o4", "chatgpt")):
        return "openai/" + m
    if ml.startswith("claude-"):
        return "anthropic/claude-opus-4.8"
    if ml.startswith("kimi"):
        # kimi-k3 is not on OpenRouter; moonshotai/kimi-k2.6 is the closest hosted id.
        return "moonshotai/kimi-k2.6"
    return "openai/gpt-5.6-luna"


class Config:
    """Configuration settings for the evaluation framework."""

    # Kimi API Settings (also accept MOONSHOT_API_KEY for compatibility)
    KIMI_API_KEY: str = os.getenv("KIMI_API_KEY") or os.getenv("MOONSHOT_API_KEY", "")
    KIMI_BASE_URL: str = os.getenv("KIMI_BASE_URL", "https://api.moonshot.cn/v1")
    KIMI_MODEL: str = os.getenv("KIMI_MODEL", "kimi-k3")

    # OpenAI API Settings (alternative)
    OPENAI_API_KEY: str = os.getenv("OPENAI_API_KEY", "")
    OPENAI_BASE_URL: str = os.getenv("OPENAI_BASE_URL", "https://api.openai.com/v1")
    OPENAI_MODEL: str = os.getenv("OPENAI_MODEL", "gpt-5.6-luna")

    # OpenRouter fallback settings (used when the primary judge key is missing)
    OPENROUTER_API_KEY: str = os.getenv("OPENROUTER_API_KEY", "")
    OPENROUTER_BASE_URL: str = "https://openrouter.ai/api/v1"
    
    # Evaluation Settings
    DEFAULT_EVALUATOR: str = os.getenv("DEFAULT_EVALUATOR", "kimi")
    MAX_RETRIES: int = int(os.getenv("MAX_RETRIES", "3"))
    REQUEST_TIMEOUT: int = int(os.getenv("REQUEST_TIMEOUT", "60"))
    
    # Test Case Settings
    TEST_CASES_DIR: str = os.path.join(os.path.dirname(__file__), "test_cases")
    
    @classmethod
    def get_evaluator_config(cls, evaluator: Optional[str] = None) -> dict:
        """Get configuration for the specified evaluator."""
        evaluator = evaluator or cls.DEFAULT_EVALUATOR
        
        if evaluator == "kimi":
            if cls.KIMI_API_KEY:
                return {
                    "api_key": cls.KIMI_API_KEY,
                    "base_url": cls.KIMI_BASE_URL,
                    "model": cls.KIMI_MODEL,
                    "type": "kimi"
                }
            if cls.OPENROUTER_API_KEY:
                return {
                    "api_key": cls.OPENROUTER_API_KEY,
                    "base_url": cls.OPENROUTER_BASE_URL,
                    "model": _openrouter_model_id(cls.KIMI_MODEL),
                    "type": "openrouter"
                }
            raise ValueError(
                "KIMI_API_KEY not configured. Set KIMI_API_KEY/MOONSHOT_API_KEY "
                "or OPENROUTER_API_KEY (fallback) in .env file."
            )
        elif evaluator == "openai":
            # gpt-5.x (incl. gpt-5.6*) needs OpenAI org-verification on the direct
            # API; when an OpenRouter key is present, prefer routing it through OR.
            prefer_openrouter = (bool(cls.OPENROUTER_API_KEY)
                                 and cls.OPENAI_MODEL.lower().startswith("gpt-5"))
            if cls.OPENAI_API_KEY and not prefer_openrouter:
                return {
                    "api_key": cls.OPENAI_API_KEY,
                    "base_url": cls.OPENAI_BASE_URL,
                    "model": cls.OPENAI_MODEL,
                    "type": "openai"
                }
            if cls.OPENROUTER_API_KEY:
                return {
                    "api_key": cls.OPENROUTER_API_KEY,
                    "base_url": cls.OPENROUTER_BASE_URL,
                    "model": _openrouter_model_id(cls.OPENAI_MODEL),
                    "type": "openrouter"
                }
            raise ValueError(
                "OPENAI_API_KEY not configured. Set OPENAI_API_KEY or "
                "OPENROUTER_API_KEY (fallback) in .env file."
            )
        else:
            raise ValueError(f"Unknown evaluator: {evaluator}. Supported: kimi, openai")