Shadowzzh

llm_agent.py

llm_agent.py
"""
LLM-based Agent using In-Context Learning with the Kimi (Moonshot) API.
This demonstrates how LLMs can generalize through reasoning without extensive training.
Default model is Kimi K3 (matching 实验 7-2 in the book); override via the
`model` argument or the MOONSHOT_MODEL environment variable.
"""

import os
import json
import time
from typing import Dict, List, Tuple, Any, Optional
from dataclasses import dataclass, asdict
import openai
from game_environment import TreasureHuntGame


def _reasoning_safe_temperature(model, requested=1.0):
    """Reasoning models (Kimi K3, GPT-5, ...) only accept temperature=1.
    Return 1 for those; otherwise the requested value so non-reasoning
    providers (Doubao, DeepSeek, older Moonshot) are unchanged."""
    m = str(model or "").lower().replace("/", "-")
    return 1 if ("kimi-k3" in m or "gpt-5" in m) else requested


def _map_model_to_openrouter(model):
    """Map a bare model id to an OpenRouter model id.
    - ids already containing '/' -> left as-is
    - gpt-*/o1-*/o3-*/o4-* -> 'openai/<id>'
    - claude-* -> anthropic Claude (opus/sonnet/haiku)
    - other native ids (kimi-*, doubao-*, ...) are NOT reliably on OpenRouter,
      so fall back to OPENROUTER_MODEL or a safe default that always works.
    """
    m = (model or "").strip()
    if "/" in m:
        return m
    ml = m.lower()
    if ml.startswith(("gpt-", "o1-", "o3-", "o4-")):
        return "openai/" + m
    if ml.startswith("claude-"):
        if "sonnet" in ml:
            return "anthropic/claude-sonnet-4.6"
        if "haiku" in ml:
            return "anthropic/claude-haiku-4.5"
        return "anthropic/claude-opus-4.8"
    if ml.startswith("kimi"):
        # kimi-k3 is not on OpenRouter; moonshotai/kimi-k2.6 is the closest hosted id.
        return "moonshotai/kimi-k2.6"
    return os.getenv("OPENROUTER_MODEL", "openai/gpt-5.6-luna")


def resolve_llm_backend(primary_key, primary_base_url, model):
    """Universal OpenRouter fallback for LLM backend resolution.

    Returns (api_key, base_url, model, using_openrouter).
    - If the primary provider key is present, behavior is unchanged.
    - Else if OPENROUTER_API_KEY is present, route through OpenRouter and map
      the model id to an OpenRouter id.
    - Else raise a clear error listing the accepted keys.
    """
    openrouter_key = os.getenv("OPENROUTER_API_KEY")
    # gpt-5.x (incl. gpt-5.6*) needs OpenAI org-verification on the direct API;
    # when an OpenRouter key is present, prefer routing these ids through it.
    if openrouter_key and str(model or "").lower().startswith("gpt-5"):
        base_url = os.getenv("OPENROUTER_BASE_URL", "https://openrouter.ai/api/v1")
        return openrouter_key, base_url, _map_model_to_openrouter(model), True
    if primary_key:
        return primary_key, primary_base_url, model, False
    if openrouter_key:
        base_url = os.getenv("OPENROUTER_BASE_URL", "https://openrouter.ai/api/v1")
        return openrouter_key, base_url, _map_model_to_openrouter(model), True
    raise ValueError(
        "No API key found. Set MOONSHOT_API_KEY (primary) or "
        "OPENROUTER_API_KEY (universal fallback)."
    )


@dataclass
class GameExperience:
    """Represents a single game interaction experience."""
    state_description: str
    action: str
    feedback: str
    reward: float
    success: bool  # Whether the action led to positive outcome


class LLMAgent:
    """
    LLM-based agent that uses in-context learning to play the game.
    Stores experiences and uses them to reason about future actions.
    """
    
    def __init__(self,
                 api_key: str = None,
                 model: str = "kimi-k3",  # Kimi K3 (see 实验 7-2)
                 base_url: str = "https://api.moonshot.cn/v1",
                 temperature: float = 0.7,
                 max_experiences: int = 50):
        """
        Initialize LLM agent with the Kimi (Moonshot) API.

        Args:
            api_key: Kimi API key (or set MOONSHOT_API_KEY env var)
            model: Model name (Moonshot/Kimi, default kimi-k3)
            base_url: API base URL
            temperature: Sampling temperature for generation
            max_experiences: Maximum number of experiences to store
        """
        # Set up API client (Moonshot primary, OpenRouter universal fallback)
        primary_key = api_key or os.getenv("MOONSHOT_API_KEY")
        self.api_key, resolved_base_url, self.model, self.using_openrouter = \
            resolve_llm_backend(primary_key, base_url, model)
        if self.using_openrouter:
            print(f"ℹ️  MOONSHOT_API_KEY not set; routing via OpenRouter (model: {self.model})")

        self.client = openai.OpenAI(
            api_key=self.api_key,
            base_url=resolved_base_url
        )
        self.temperature = temperature
        
        # Experience memory for in-context learning
        self.experiences: List[GameExperience] = []
        self.max_experiences = max_experiences
        
        # Statistics
        self.episode_rewards = []
        self.episode_lengths = []
        self.victories = 0
        self.total_episodes = 0
        self.api_calls = 0
        self.total_tokens = 0
    
    def _build_context(self, current_state: str, available_actions: List[str]) -> str:
        """
        Build context for the LLM including task description and past experiences.
        This is the key to in-context learning.
        """
        context = []
        
        # Task description
        context.append("""You are playing a text-based treasure hunt game. Your goal is to find and collect the dragon's treasure.

The game has hidden mechanics that you need to discover through experience:
- Certain items may be required to unlock doors or defeat guards
- Items might combine to create better items
- Different weapons have different effectiveness

You should reason about what you've learned from past experiences to make better decisions.""")
        
        # Add relevant past experiences
        if self.experiences:
            context.append("\n=== PAST EXPERIENCES ===")
            context.append("Here are some experiences from previous attempts that might help you:")
            
            # Group experiences by pattern
            successful_patterns = []
            failed_patterns = []
            
            for exp in self.experiences[-self.max_experiences:]:
                exp_text = f"State: {exp.state_description[:200]}...\nAction: {exp.action}\nResult: {exp.feedback}\nReward: {exp.reward:.1f}"
                
                if exp.success:
                    successful_patterns.append(exp_text)
                else:
                    failed_patterns.append(exp_text)
            
            if successful_patterns:
                context.append("\n** Successful actions:")
                for pattern in successful_patterns[-10:]:  # Last 10 successful
                    context.append(pattern)
            
            if failed_patterns:
                context.append("\n** Failed actions to avoid:")
                for pattern in failed_patterns[-5:]:  # Last 5 failed
                    context.append(pattern)
        
        # Current situation
        context.append("\n=== CURRENT SITUATION ===")
        context.append(current_state)
        context.append(f"\nAvailable actions: {', '.join(available_actions)}")
        
        return "\n".join(context)
    
    def _build_prompt(self, context: str) -> str:
        """Build the full prompt for the LLM."""
        prompt = f"""{context}

Based on your understanding of the game mechanics from past experiences and the current situation, reason step-by-step about what action to take:

1. What have you learned from past experiences that applies here?
2. What is your current goal or sub-goal?
3. Which available action best helps achieve that goal?

Think through this carefully, then provide your chosen action.

IMPORTANT: Your response must end with exactly one line starting with "ACTION:" followed by one of the available actions listed above.

Example format:
[Your reasoning here...]
ACTION: take red key
"""
        return prompt
    
    def choose_action(self, game: TreasureHuntGame, verbose: bool = True) -> str:
        """
        Choose an action using LLM reasoning with in-context learning.
        """
        # Get current state and available actions
        state_description = game.get_state_description()
        available_actions = game.get_available_actions()
        
        if not available_actions:
            return "look around"
        
        # Build context with past experiences
        context = self._build_context(state_description, available_actions)
        prompt = self._build_prompt(context)
        
        if verbose:
            print("\n" + "="*60)
            print("LLM DECISION PROCESS")
            print("="*60)
            print(f"📊 Experiences in memory: {len(self.experiences)}")
            print(f"🎮 Current room: {game.current_room.name}")
            print(f"🎯 Available actions: {len(available_actions)}")
            
            # Show some recent successful experiences if any
            successful = [e for e in self.experiences if e.success]
            if successful:
                print(f"\n💡 Recent successful patterns learned:")
                for exp in successful[-3:]:
                    print(f"   • {exp.action} → +{exp.reward:.1f} reward")
        
        try:
            print("\n🤔 LLM is thinking...")
            
            # Call Kimi K3 API.
            # Kimi K3 is a REASONING model: its chain-of-thought is billed as
            # completion tokens (returned separately in message.reasoning_content),
            # and the final answer in message.content is only emitted AFTER the
            # reasoning finishes. A small max_tokens can therefore be consumed
            # entirely by reasoning, truncating content to empty and breaking the
            # ACTION parse. Keep a generous budget so the ACTION line survives.
            response = self.client.chat.completions.create(
                model=self.model,
                messages=[
                    {"role": "system", "content": "You are an intelligent game-playing agent that learns from experience."},
                    {"role": "user", "content": prompt}
                ],
                temperature=_reasoning_safe_temperature(self.model, self.temperature),
                max_tokens=2048
            )
            
            self.api_calls += 1
            if hasattr(response.usage, 'total_tokens'):
                self.total_tokens += response.usage.total_tokens
            
            # Extract action from response (final answer lives in .content;
            # reasoning models expose their thinking separately in
            # .reasoning_content, which we don't need for action parsing).
            response_text = response.choices[0].message.content or ""
            
            if verbose:
                print("\n📝 LLM Reasoning:")
                print("-" * 40)
                # Show the reasoning part (before ACTION:)
                reasoning_lines = []
                for line in response_text.split('\n'):
                    if line.startswith("ACTION:"):
                        break
                    if line.strip():
                        reasoning_lines.append(line)
                
                # Show last few lines of reasoning
                for line in reasoning_lines[-5:]:
                    print(f"  {line[:100]}...")  # Truncate long lines
                print("-" * 40)
            
            # Parse action from response
            lines = response_text.strip().split('\n')
            for line in reversed(lines):
                if line.startswith("ACTION:"):
                    action = line[7:].strip()
                    
                    # Validate action
                    if action in available_actions:
                        if verbose:
                            print(f"\n✅ Chosen action: {action}")
                        return action
                    else:
                        # Try to find closest matching action
                        action_lower = action.lower()
                        for available in available_actions:
                            if available.lower() == action_lower:
                                if verbose:
                                    print(f"\n✅ Chosen action (corrected): {available}")
                                return available
            
            # Fallback if no valid action found
            print(f"⚠️ Warning: Could not parse valid action from LLM response. Using fallback.")
            return available_actions[0]
            
        except Exception as e:
            print(f"❌ Error calling LLM API: {e}")
            # Fallback to first available action
            return available_actions[0]
    
    def update_experience(self, state: str, action: str, feedback: str, reward: float):
        """
        Store an experience for future in-context learning.
        """
        # Determine if action was successful based on reward
        success = reward > 0
        
        experience = GameExperience(
            state_description=state,
            action=action,
            feedback=feedback,
            reward=reward,
            success=success
        )
        
        self.experiences.append(experience)
        
        # Keep only recent experiences to manage context length
        if len(self.experiences) > self.max_experiences * 2:
            # Keep a mix of successful and failed experiences
            successful = [e for e in self.experiences if e.success]
            failed = [e for e in self.experiences if not e.success]
            
            # Keep recent ones and some diverse older ones
            self.experiences = (
                successful[-self.max_experiences:] + 
                failed[-self.max_experiences//2:]
            )[-self.max_experiences:]
    
    def play_episode(self, game: TreasureHuntGame, verbose: bool = True) -> Tuple[float, int, bool]:
        """
        Play one episode of the game.
        """
        game.reset()
        total_reward = 0
        steps = 0
        trajectory = []
        
        if verbose:
            print("\n" + "🎮"*30)
            print("STARTING NEW GAME EPISODE")
            print("🎮"*30)
        
        while not game.game_over:
            if verbose:
                print(f"\n{'='*60}")
                print(f"STEP {steps + 1}")
                print(f"{'='*60}")
                
                # Show current game state
                print("\n📍 Current State:")
                state_lines = game.get_state_description().split('\n')
                for line in state_lines:
                    if line.strip():
                        print(f"  {line}")
            
            # Get state before action
            state_before = game.get_state_description()
            
            # Choose action using LLM
            action = self.choose_action(game, verbose=verbose)
            
            # Execute action
            feedback, reward, done = game.execute_action(action)
            
            # Store experience
            self.update_experience(state_before, action, feedback, reward)
            
            # Record trajectory
            trajectory.append({
                "step": steps + 1,
                "action": action,
                "reward": reward,
                "feedback": feedback
            })
            
            total_reward += reward
            steps += 1
            
            if verbose:
                print(f"\n🎯 Action Result:")
                print(f"  Feedback: {feedback}")
                if reward > 0:
                    print(f"  Reward: ✨ +{reward:.1f}")
                else:
                    print(f"  Reward: 📉 {reward:.1f}")
                print(f"  Total reward so far: {total_reward:.1f}")
                
                # Add a pause between steps for readability
                if not done:
                    print("\n" + "."*60)
        
        # Update statistics
        self.episode_rewards.append(total_reward)
        self.episode_lengths.append(steps)
        if game.victory:
            self.victories += 1
        self.total_episodes += 1
        
        if verbose:
            print("\n" + "🏁"*30)
            if game.victory:
                print("🎉 VICTORY! The LLM found the treasure!")
            else:
                print("💀 GAME OVER! Better luck next time.")
            print(f"  Final Score: {total_reward:.1f}")
            print(f"  Total Steps: {steps}")
            print(f"  API Calls Used: {self.api_calls}")
            print("🏁"*30)
        
        return total_reward, steps, game.victory
    
    def train(self, num_episodes: int = 20, verbose: bool = True, stochastic: bool = False) -> Dict[str, Any]:
        """
        'Train' the agent through in-context learning over multiple episodes.
        Note: Unlike traditional RL, there's no explicit training - just experience accumulation.
        
        Args:
            num_episodes: Number of episodes to play
            verbose: Whether to print details
            stochastic: Whether to use stochastic environment
        """
        game = TreasureHuntGame(stochastic=stochastic)
        
        print("\n" + "🚀"*30)
        print("LLM IN-CONTEXT LEARNING EXPERIMENT")
        print("🚀"*30)
        print(f"\n📝 Will play {num_episodes} episodes to learn the game")
        print("🧠 The LLM learns by accumulating experiences in context")
        print("⚡ Each decision shows the full reasoning process")
        
        for episode in range(num_episodes):
            print(f"\n\n{'🎯'*30}")
            print(f"EPISODE {episode + 1} of {num_episodes}")
            print(f"{'🎯'*30}")
            print(f"📚 Experiences accumulated so far: {len(self.experiences)}")
            
            # Show full process for first 3 episodes, then reduce verbosity
            show_full = verbose and (episode < 3 or episode == num_episodes - 1)
            
            if not show_full and verbose:
                print("\n(Reducing verbosity for middle episodes to save space...)")
            
            reward, steps, victory = self.play_episode(game, verbose=show_full)
            
            if not show_full:
                # Still show summary even when not fully verbose
                print(f"\n📊 Episode {episode + 1} Summary:")
                print(f"  Result: {'🎉 Victory!' if victory else '💀 Failed'}")
                print(f"  Total Reward: {reward:.2f}")
                print(f"  Steps Taken: {steps}")
                print(f"  Total API Calls So Far: {self.api_calls}")
            
            # Show learning progress
            if len(self.episode_rewards) >= 3:
                recent_victories = sum(1 for r in self.episode_rewards[-3:] if r > 50)
                recent_avg = sum(self.episode_rewards[-3:]) / 3
                print(f"\n📈 Recent Performance (last 3 episodes):")
                print(f"  Victories: {recent_victories}/3")
                print(f"  Average Reward: {recent_avg:.2f}")
            
            # Add delay to respect rate limits
            if episode < num_episodes - 1:
                print("\n⏳ Waiting 1 second for API rate limits...")
                time.sleep(1)
        
        return {
            "total_episodes": self.total_episodes,
            "total_victories": self.victories,
            "victory_rate": self.victories / self.total_episodes if self.total_episodes > 0 else 0,
            "total_api_calls": self.api_calls,
            "total_tokens": self.total_tokens,
            "experiences_collected": len(self.experiences),
            "episode_rewards": self.episode_rewards,
            "episode_lengths": self.episode_lengths
        }
    
    def evaluate(self, num_episodes: int = 10, verbose: bool = False, stochastic: bool = False) -> Dict[str, Any]:
        """
        Evaluate the agent's performance using accumulated experiences.
        
        Args:
            num_episodes: Number of episodes to evaluate
            verbose: Whether to print details
            stochastic: Whether to use stochastic environment
        """
        game = TreasureHuntGame(stochastic=stochastic)
        eval_rewards = []
        eval_lengths = []
        eval_victories = 0
        
        for episode in range(num_episodes):
            reward, steps, victory = self.play_episode(game, verbose=verbose)
            
            eval_rewards.append(reward)
            eval_lengths.append(steps)
            if victory:
                eval_victories += 1
            
            if verbose:
                print(f"Episode {episode + 1}: Reward={reward:.2f}, Steps={steps}, Victory={victory}")
        
        return {
            "num_episodes": num_episodes,
            "victories": eval_victories,
            "victory_rate": eval_victories / num_episodes if num_episodes else 0.0,
            "avg_reward": sum(eval_rewards) / len(eval_rewards) if eval_rewards else 0.0,
            "avg_length": sum(eval_lengths) / len(eval_lengths) if eval_lengths else 0.0,
            "total_api_calls": self.api_calls,
            "experiences_used": len(self.experiences)
        }
    
    def save_experiences(self, filepath: str):
        """Save experiences to file for analysis."""
        data = {
            "experiences": [asdict(exp) for exp in self.experiences],
            "statistics": {
                "total_episodes": self.total_episodes,
                "victories": self.victories,
                "api_calls": self.api_calls,
                "total_tokens": self.total_tokens
            }
        }
        
        with open(filepath, 'w') as f:
            json.dump(data, f, indent=2)
    
    def load_experiences(self, filepath: str):
        """Load experiences from file."""
        with open(filepath, 'r') as f:
            data = json.load(f)
        
        self.experiences = [
            GameExperience(**exp) for exp in data["experiences"]
        ]
        
        stats = data.get("statistics", {})
        self.total_episodes = stats.get("total_episodes", 0)
        self.victories = stats.get("victories", 0)
        self.api_calls = stats.get("api_calls", 0)
        self.total_tokens = stats.get("total_tokens", 0)