Shadowzzh

compare_models.py

compare_models.py
# -*- coding: utf-8 -*-
"""
Compare baseline → pretrained → finetuned Korean Mistral models (3-way comparison)
Shows progression from original model to final Korean-capable model
"""

import argparse

# 说明:unsloth / torch 等重型依赖在函数内按需导入,
# 这样 `python compare_models.py --help` 无需 GPU 环境即可查看参数。

# ANSI color codes for colored output
class Colors:
    HEADER = '\033[95m'
    BLUE = '\033[94m'
    CYAN = '\033[96m'
    GREEN = '\033[92m'
    YELLOW = '\033[93m'
    RED = '\033[91m'
    ENDC = '\033[0m'
    BOLD = '\033[1m'
    UNDERLINE = '\033[4m'

def print_section(title, color=Colors.CYAN):
    """Print a colored section header"""
    print(f"\n{color}{Colors.BOLD}{'='*80}")
    print(f"{title}")
    print(f"{'='*80}{Colors.ENDC}\n")

def load_baseline_model(base_model="unsloth/mistral-7b-v0.3", max_seq_length=2048):
    """Load the original Mistral model (before any training)"""
    from unsloth import FastLanguageModel
    model, tokenizer = FastLanguageModel.from_pretrained(
        model_name=base_model,
        max_seq_length=max_seq_length,
        dtype=None,
        load_in_4bit=True,
    )
    FastLanguageModel.for_inference(model)
    return model, tokenizer

def load_model(model_path, max_seq_length=2048):
    """Load a trained LoRA model"""
    from unsloth import FastLanguageModel
    model, tokenizer = FastLanguageModel.from_pretrained(
        model_name=model_path,
        max_seq_length=max_seq_length,
        dtype=None,
        load_in_4bit=True,
    )
    FastLanguageModel.for_inference(model)
    return model, tokenizer

def generate_text(model, tokenizer, prompt, max_new_tokens=150, temperature=0.3):
    """Generate text without streaming"""
    inputs = tokenizer([prompt], return_tensors="pt").to("cuda")
    outputs = model.generate(
        **inputs,
        max_new_tokens=max_new_tokens,
        use_cache=True,
        do_sample=True,
        temperature=temperature,
        pad_token_id=tokenizer.eos_token_id,
    )
    generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
    # Remove the prompt from the output
    response = generated_text[len(prompt):].strip()
    return response

def compare_on_prompt(baseline_model, baseline_tokenizer,
                     pretrained_model, pretrained_tokenizer, 
                     finetuned_model, finetuned_tokenizer,
                     prompt, test_name, prompt_translation=None,
                     max_new_tokens=150, temperature=0.3):
    """Compare three models on the same prompt"""
    
    print(f"\n{Colors.BOLD}{'='*80}")
    print(f"{test_name}")
    print(f"{'='*80}{Colors.ENDC}")
    
    if prompt_translation:
        print(f"{Colors.CYAN}Prompt (Translation): {prompt_translation}{Colors.ENDC}\n")
    
    print(f"{Colors.YELLOW}Generating from BASELINE model (original Mistral)...{Colors.ENDC}")
    baseline_output = generate_text(
        baseline_model, baseline_tokenizer, prompt, 
        max_new_tokens, temperature
    )
    
    print(f"{Colors.YELLOW}Generating from PRETRAINED model (after Korean training)...{Colors.ENDC}")
    pretrained_output = generate_text(
        pretrained_model, pretrained_tokenizer, prompt, 
        max_new_tokens, temperature
    )
    
    print(f"{Colors.YELLOW}Generating from FINETUNED model (after instruction tuning)...{Colors.ENDC}")
    finetuned_output = generate_text(
        finetuned_model, finetuned_tokenizer, prompt,
        max_new_tokens, temperature
    )
    
    # Display all three outputs
    print(f"\n{Colors.RED}┌─ BASELINE MODEL (Original Mistral) ───────────────────────────────┐{Colors.ENDC}")
    print(f"{Colors.RED}│{Colors.ENDC}")
    for line in baseline_output.split('\n'):
        print(f"{Colors.RED}│{Colors.ENDC} {line}")
    print(f"{Colors.RED}│{Colors.ENDC}")
    print(f"{Colors.RED}└────────────────────────────────────────────────────────────────────┘{Colors.ENDC}\n")
    
    print(f"{Colors.GREEN}┌─ PRETRAINED MODEL (After Korean Wikipedia) ───────────────────────┐{Colors.ENDC}")
    print(f"{Colors.GREEN}│{Colors.ENDC}")
    for line in pretrained_output.split('\n'):
        print(f"{Colors.GREEN}│{Colors.ENDC} {line}")
    print(f"{Colors.GREEN}│{Colors.ENDC}")
    print(f"{Colors.GREEN}└────────────────────────────────────────────────────────────────────┘{Colors.ENDC}\n")
    
    print(f"{Colors.CYAN}┌─ FINETUNED MODEL (After Instruction Tuning) ──────────────────────┐{Colors.ENDC}")
    print(f"{Colors.CYAN}│{Colors.ENDC}")
    for line in finetuned_output.split('\n'):
        print(f"{Colors.CYAN}│{Colors.ENDC} {line}")
    print(f"{Colors.CYAN}│{Colors.ENDC}")
    print(f"{Colors.CYAN}└────────────────────────────────────────────────────────────────────┘{Colors.ENDC}\n")

def parse_args():
    parser = argparse.ArgumentParser(
        description="对比韩语 Mistral 的三个阶段模型:基础模型 → 继续预训练 → 指令微调。"
                    "在同一批中韩英提示上并排生成,直观展示韩语能力的提升与英语能力的保留。",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    )
    parser.add_argument("--base_model", type=str, default="unsloth/mistral-7b-v0.3",
                        help="基础(未训练)模型名称")
    parser.add_argument("--pretrained_path", type=str, default="lora_model_pretrained",
                        help="继续预训练后保存的 LoRA 模型目录")
    parser.add_argument("--finetuned_path", type=str, default="lora_model",
                        help="指令微调后保存的最终 LoRA 模型目录")
    parser.add_argument("--max_seq_length", type=int, default=2048,
                        help="最大序列长度")
    parser.add_argument("--max_new_tokens", type=int, default=150,
                        help="每次生成的最大 token 数")
    parser.add_argument("--temperature", type=float, default=0.3,
                        help="采样温度(越低越确定)")
    return parser.parse_args()


def main():
    args = parse_args()

    print_section("🔬 KOREAN MISTRAL 3-WAY MODEL COMPARISON", Colors.HEADER)
    print(f"{Colors.YELLOW}This script compares three model stages:{Colors.ENDC}")
    print(f"  1. {Colors.RED}Baseline{Colors.ENDC} - Original Mistral (no Korean training)")
    print(f"  2. {Colors.GREEN}Pretrained{Colors.ENDC} - After Korean Wikipedia training")
    print(f"  3. {Colors.CYAN}Finetuned{Colors.ENDC} - After instruction tuning")
    print(f"\n{Colors.CYAN}Generation settings: temperature=0.3, do_sample=True (no repetition_penalty){Colors.ENDC}\n")
    
    # Load all three models
    print_section("📥 LOADING MODELS", Colors.BLUE)
    
    print(f"{Colors.YELLOW}Loading baseline model (original Mistral v0.3)...{Colors.ENDC}")
    baseline_model, baseline_tokenizer = load_baseline_model()
    print(f"{Colors.GREEN}✓ Baseline model loaded{Colors.ENDC}")
    
    print(f"\n{Colors.YELLOW}Loading pretrained model (after Korean pretraining)...{Colors.ENDC}")
    pretrained_model, pretrained_tokenizer = load_model("lora_model_pretrained")
    print(f"{Colors.GREEN}✓ Pretrained model loaded{Colors.ENDC}")
    
    print(f"\n{Colors.YELLOW}Loading finetuned model (after instruction tuning)...{Colors.ENDC}")
    finetuned_model, finetuned_tokenizer = load_model("lora_model")
    print(f"{Colors.GREEN}✓ Finetuned model loaded{Colors.ENDC}")
    
    # Define prompts
    wikipedia_prompt_korean = """위키피디아 기사
### 제목: {}

### 기사:
{}"""

    wikipedia_prompt_english = """Wikipedia Article
### Title: {}

### Article:
{}"""

    alpaca_prompt_korean = """다음은 작업을 설명하는 명령입니다. 요청을 적절하게 완료하는 응답을 작성하세요.

### 지침:
{}

### 응답:
{}"""

    alpaca_prompt_english = """Below is an instruction that describes a task. Write a response that appropriately completes the request.

### Instruction:
{}

### Response:
{}"""
    
    print_section("🧪 RUNNING 3-WAY COMPARISONS", Colors.CYAN)
    
    # Test 1: Korean Wikipedia
    compare_on_prompt(
        baseline_model, baseline_tokenizer,
        pretrained_model, pretrained_tokenizer,
        finetuned_model, finetuned_tokenizer,
        wikipedia_prompt_korean.format("인공지능", ""),
        "Test 1: Korean Wikipedia - Artificial Intelligence (인공지능)",
        "Wikipedia Article / Title: Artificial Intelligence / Article:",
        max_new_tokens=args.max_new_tokens, temperature=args.temperature
    )
    
    # Test 2: English Wikipedia - Preservation Check
    compare_on_prompt(
        baseline_model, baseline_tokenizer,
        pretrained_model, pretrained_tokenizer,
        finetuned_model, finetuned_tokenizer,
        wikipedia_prompt_english.format("Artificial Intelligence", ""),
        "Test 2: English Wikipedia - Artificial Intelligence (Preservation Check)",
        None,
        max_new_tokens=args.max_new_tokens, temperature=args.temperature
    )
    
    # Test 3: Korean Instruction - Kimchi
    compare_on_prompt(
        baseline_model, baseline_tokenizer,
        pretrained_model, pretrained_tokenizer,
        finetuned_model, finetuned_tokenizer,
        alpaca_prompt_korean.format("한국의 전통 음식인 김치에 대해 설명하세요.", ""),
        "Test 3: Korean Instruction - Explain Kimchi",
        "Instruction: Explain about kimchi, a traditional Korean food. / Response:",
        max_new_tokens=args.max_new_tokens, temperature=args.temperature
    )
    
    # Test 4: Korean Instruction - Seoul
    compare_on_prompt(
        baseline_model, baseline_tokenizer,
        pretrained_model, pretrained_tokenizer,
        finetuned_model, finetuned_tokenizer,
        alpaca_prompt_korean.format("대한민국의 수도인 서울에 대해 간단히 소개해주세요.", ""),
        "Test 4: Korean Instruction - Introduce Seoul",
        "Instruction: Briefly introduce Seoul, the capital of South Korea. / Response:",
        max_new_tokens=args.max_new_tokens, temperature=args.temperature
    )
    
    # Test 5: English Instruction - Preservation Check
    compare_on_prompt(
        baseline_model, baseline_tokenizer,
        pretrained_model, pretrained_tokenizer,
        finetuned_model, finetuned_tokenizer,
        alpaca_prompt_english.format("Explain about Thanksgiving turkey, a traditional American food.", ""),
        "Test 5: English Instruction - Thanksgiving Turkey (Preservation Check)",
        None,
        max_new_tokens=args.max_new_tokens, temperature=args.temperature
    )
    
    print_section("📊 COMPARISON COMPLETE", Colors.GREEN)
    
    print(f"{Colors.CYAN}{'='*80}")
    print(f"💡 What to Look For:")
    print(f"{'='*80}{Colors.ENDC}")
    
    print(f"\n{Colors.RED}Baseline Model (Red boxes - Original Mistral):{Colors.ENDC}")
    print(f"  • Korean: Should be POOR - repetitive, nonsensical")
    print(f"  • English: Should be GOOD - this is the starting point")
    print(f"  • Shows what model knows BEFORE any Korean training")
    
    print(f"\n{Colors.GREEN}Pretrained Model (Green boxes - After Korean Wikipedia):{Colors.ENDC}")
    print(f"  • Korean: Should show IMPROVED fluency and vocabulary")
    print(f"  • Better Korean sentence structure")
    print(f"  • Weak instruction-following (only learned language, not how to follow instructions)")
    print(f"  • English: Should REMAIN strong (no catastrophic forgetting)")
    
    print(f"\n{Colors.CYAN}Finetuned Model (Cyan boxes - After Instruction Tuning):{Colors.ENDC}")
    print(f"  • Korean: Should be FLUENT with GOOD instruction-following")
    print(f"  • More structured and complete responses")
    print(f"  • Directly answers questions")
    print(f"  • English: Should REMAIN strong")
    
    print(f"\n{Colors.YELLOW}Key Progression to Observe:{Colors.ENDC}")
    print(f"  📊 Korean Quality:    {Colors.RED}Poor{Colors.ENDC} → {Colors.GREEN}Better{Colors.ENDC} → {Colors.CYAN}Best{Colors.ENDC}")
    print(f"  📊 Instruction:       {Colors.RED}Weak{Colors.ENDC} → {Colors.GREEN}Weak{Colors.ENDC} → {Colors.CYAN}Strong{Colors.ENDC}")
    print(f"  📊 English Quality:   {Colors.RED}Good{Colors.ENDC} → {Colors.GREEN}Good{Colors.ENDC} → {Colors.CYAN}Good{Colors.ENDC}")
    print(f"  📊 Repetition:        {Colors.RED}High{Colors.ENDC} → {Colors.GREEN}Medium{Colors.ENDC} → {Colors.CYAN}Low{Colors.ENDC}")
    
    print(f"\n{Colors.YELLOW}This demonstrates:{Colors.ENDC}")
    print(f"  ✓ Continued pretraining successfully teaches new language (Korean)")
    print(f"  ✓ Instruction tuning teaches how to follow instructions in the new language")
    print(f"  ✓ English capability is preserved throughout (no catastrophic forgetting)")
    print(f"  ✓ Both Wikipedia and Instruction tasks show English preservation")
    print(f"  ✓ Two-stage approach is necessary: language first, then instruction-following")
    print(f"\n{Colors.CYAN}💡 Note: Compare the English tests (Tests 2 & 5) across all three models.")
    print(f"All three should perform similarly well, proving no English degradation.{Colors.ENDC}")
    print()

if __name__ == "__main__":
    main()