ClassyGitshadowzzh

demo.py

demo.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
实验 5-2:用代码生成工具提升逻辑思考能力

对比在三种模式下求解「骑士与无赖」(Knights & Knaves) 谜题的准确率:

  1) 纯思考(pure)      —— LLM 仅靠自然语言链式推理直接给出答案;
  2) 代码辅助(code)    —— LLM 配备 Code Interpreter(预装 python-constraint),
                          把谜题形式化为约束满足问题(CSP),调用求解器搜索答案;
  3) 约束求解(solver)  —— 【离线,无需 API】直接用 python-constraint 求解结构化
                          陈述,作为确定性基线(理论上 100% 正确)。

结论预期:约束求解把逻辑推理外包给确定性求解器,准确率应达 90%+,
且显著高于纯思考模式(纯思考在多人、含计数/自指的谜题上容易出错)。

用法:
    # 离线约束求解基线(不花钱、不联网,演示核心论点):
    python demo.py --mode solver

    # LLM 对照实验(需要 OPENAI_API_KEY):
    export OPENAI_API_KEY=sk-...
    python demo.py                       # 默认 both:跑 纯思考 vs 代码辅助 全部题目
    python demo.py --mode pure           # 只跑纯思考
    python demo.py --limit 4             # 只跑前 4 题(省钱冒烟测试)
    python demo.py --max-people 3        # 只跑不超过 3 人的谜题(按难度筛选)
    python demo.py --model gpt-4o-mini    # 指定模型(默认 gpt-4o-mini)
    python demo.py --puzzles my.json     # 换一份谜题数据集
"""
import argparse
import json
import os
import re
import sys

from csp_solver import solve_labeled
from sandbox import run_python

# ---- 读取 .env(如果存在)。避免额外依赖,手写一个极简解析器。----
def _load_dotenv(path=".env"):
    if not os.path.exists(path):
        return
    with open(path, encoding="utf-8") as f:
        for line in f:
            line = line.strip()
            if not line or line.startswith("#") or "=" not in line:
                continue
            k, v = line.split("=", 1)
            os.environ.setdefault(k.strip(), v.strip().strip('"').strip("'"))


_load_dotenv()

MODEL = os.environ.get("MODEL", "gpt-4o-mini")

# --- 通用 OpenRouter 兜底:无直连 key 时自动改走 OpenRouter ---
OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1"


def map_model_to_openrouter(model: str) -> str:
    """把直连模型名映射为 OpenRouter 上的 id(非可映射 id 统一兜底到当前廉价旗舰)。"""
    if not model or "/" in model:
        return model or "openai/gpt-5.6-luna"
    m = model.lower()
    if m.startswith(("gpt-", "o1", "o3", "o4")):
        return "openai/" + model
    if m.startswith("claude"):
        if "haiku" in m:
            return "anthropic/claude-haiku-4.5"
        if "sonnet" in m:
            return "anthropic/claude-sonnet-4.6"
        return "anthropic/claude-opus-4.8"
    if m.startswith("gemini"):
        return "google/" + model
    return "openai/gpt-5.6-luna"


def build_client_and_model():
    """构造 OpenAI 客户端并返回 (client, model)。

    - 有 OPENAI_API_KEY:直连(默认模型 gpt-4o-mini 是普通 gpt id,可直连 OpenAI)。
      仅当模型是 gpt-5.x 且同时设置了 OPENROUTER_API_KEY 时才优先走 OpenRouter
      (直连 gpt-5.x 需组织实名认证)。
    - 无 OPENAI_API_KEY 但有 OPENROUTER_API_KEY:整体改走 OpenRouter。
    """
    from openai import OpenAI
    global MODEL
    api_key = os.environ.get("OPENAI_API_KEY")
    base_url = os.environ.get("OPENAI_BASE_URL")
    orkey = os.environ.get("OPENROUTER_API_KEY")
    prefer_or = bool(orkey) and (MODEL or "").lower().startswith("gpt-5")
    if prefer_or or (not api_key and orkey):
        api_key, base_url, MODEL = orkey, OPENROUTER_BASE_URL, map_model_to_openrouter(MODEL)
    kw = {"api_key": api_key, "timeout": 60.0, "max_retries": 3}
    if base_url:
        kw["base_url"] = base_url
    return OpenAI(**kw), MODEL


def _reasoning(model: str) -> bool:
    """推理模型(gpt-5 / o 系列 / *thinking 等)不接受 temperature=0。"""
    return any(k in (model or "").lower()
               for k in ("gpt-5", "o1", "o3", "o4", "thinking", "reasoner", "kimi-k3"))

# run_python 工具的 function calling 定义
TOOLS = [{
    "type": "function",
    "function": {
        "name": "run_python",
        "description": (
            "在预装了 python-constraint 库的沙箱中执行 Python 代码,返回 stdout/stderr。"
            "用它把逻辑谜题建模为约束满足问题并求解。记得用 print() 打印结果。"
        ),
        "parameters": {
            "type": "object",
            "properties": {
                "code": {"type": "string", "description": "要执行的完整 Python 代码"}
            },
            "required": ["code"],
        },
    },
}]

ANSWER_HINT = (
    '推理结束后,请在最后单独用一行输出 JSON 形式的最终答案,'
    '键为每个居民的名字,值为 "knight" 或 "knave",例如:'
    '{"A": "knight", "B": "knave"}'
)

PURE_SYSTEM = (
    "你是逻辑推理专家。在「骑士与无赖」谜题中,骑士永远说真话,无赖永远说假话。"
    "请仅凭自己的推理,逐步分析每位居民的身份,找出满足所有陈述的唯一解。\n" + ANSWER_HINT
)

CODE_SYSTEM = (
    "你是逻辑推理专家,擅长把谜题转化为形式化约束并用代码求解。"
    "在「骑士与无赖」谜题中,骑士永远说真话,无赖永远说假话。\n"
    "请务必使用 run_python 工具,用 python-constraint 库把谜题建模为约束满足问题(CSP)来求解。\n\n"
    "【最关键的建模规则】不要把某人的陈述直接当成事实约束!"
    "正确做法是对每位居民 X 加一条【双条件(等价)约束】:\n"
    "    X 的布尔值  ==  (X 那句话在语义上为真)\n"
    "含义:X 是骑士(True) 当且仅当 他的话为真;X 是无赖(False) 当且仅当 他的话为假。\n"
    "这条规则对每一句话都适用,包括计数类('恰好有两个骑士')和自指类('我和 B 同类')的陈述——"
    "都要写成 `X == (那句话的真值表达式)`,绝不能把 `(那句话的真值表达式)` 单独当作硬约束。\n\n"
    "示例(设 True=骑士):\n"
    "    from constraint import Problem\n"
    "    p = Problem()\n"
    "    for name in ['A','B','C']:\n"
    "        p.addVariable(name, [True, False])\n"
    "    # A 说'我们中恰好有一个骑士'  ->  A == ( (A+B+C)==1 )\n"
    "    p.addConstraint(lambda a,b,c: a == ((a+b+c)==1), ['A','B','C'])\n"
    "    # B 说'C 是无赖'             ->  B == (not C)\n"
    "    p.addConstraint(lambda b,c: b == (not c), ['B','C'])\n"
    "    # C 说'我和 A 是同一类人'     ->  C == (C == A)\n"
    "    p.addConstraint(lambda a,c: c == (c == a), ['A','C'])\n"
    "    for s in p.getSolutions():\n"
    "        print({k:('knight' if v else 'knave') for k,v in s.items()})\n\n"
    "步骤:1) 每人一个布尔变量;2) 每句话写成上面的双条件约束;"
    "3) 调用 getSolutions() 枚举所有解并 print。\n"
    "最终答案必须严格采用求解器打印出的解,不要用自己的直觉去推翻它。"
    "若求解器输出为空,说明约束建错了(很可能漏了双条件),请检查并重跑。\n" + ANSWER_HINT
)


def parse_answer(text, names):
    """从模型输出里提取最后一个形如 {name: knight/knave} 的 JSON 答案。"""
    norm = {"knight": "knight", "knave": "knave", "骑士": "knight", "无赖": "knave"}
    # 找出所有 {...} 片段,从后往前尝试解析
    for m in reversed(list(re.finditer(r"\{[^{}]*\}", text))):
        try:
            obj = json.loads(m.group(0))
        except json.JSONDecodeError:
            continue
        if not isinstance(obj, dict):
            continue
        got = {}
        for n in names:
            if n not in obj:
                break
            v = str(obj[n]).strip().lower()
            v = norm.get(v, norm.get(str(obj[n]).strip(), None))
            if v is None:
                break
            got[n] = v
        else:
            return got
    return None


def call_model(client, system, user, use_tools):
    """跑一轮对话(含可能的多次工具调用),返回 (最终文本, 使用的代码列表)。"""
    messages = [{"role": "system", "content": system},
                {"role": "user", "content": user}]
    codes = []
    for _ in range(8):  # 最多 8 轮,防止无限循环
        kwargs = (dict(model=MODEL, messages=messages, temperature=1, max_tokens=8192)
                  if _reasoning(MODEL)
                  else dict(model=MODEL, messages=messages, temperature=0))
        if use_tools:
            kwargs.update(tools=TOOLS, tool_choice="auto")
        resp = client.chat.completions.create(**kwargs)
        msg = resp.choices[0].message
        if use_tools and msg.tool_calls:
            messages.append(msg)
            for tc in msg.tool_calls:
                try:
                    code = json.loads(tc.function.arguments).get("code", "")
                except json.JSONDecodeError:
                    code = ""
                codes.append(code)
                result = run_python(code)
                messages.append({"role": "tool", "tool_call_id": tc.id,
                                 "content": result})
            continue
        return msg.content or "", codes
    return "", codes


def run_mode(client, puzzles, mode):
    """跑一种 LLM 模式(pure/code),返回逐题记录列表。"""
    system = CODE_SYSTEM if mode == "code" else PURE_SYSTEM
    records = []
    for p in puzzles:
        text, codes = call_model(client, system, p["description"], mode == "code")
        pred = parse_answer(text, p["names"])
        correct = pred == p["solution"]
        records.append(dict(id=p["id"], num=p["num_people"], pred=pred,
                            gold=p["solution"], correct=correct,
                            codes=codes, text=text))
        mark = "✓" if correct else "✗"
        print(f"  [{mode:6}] {p['id']} ({p['num_people']}人) {mark}  "
              f"预测={pred}")
    return records


def run_solver(puzzles):
    """离线约束求解模式:直接用 python-constraint 求解结构化陈述,无需 LLM/API。"""
    records = []
    for p in puzzles:
        struct = p.get("statements_struct")
        if not struct:
            sys.exit(f"错误:谜题 {p['id']} 缺少 statements_struct 字段,"
                     "请用新版 build_puzzles.py 重新生成 puzzles.json。")
        sols = solve_labeled(p["names"], struct)
        pred = sols[0] if len(sols) == 1 else None
        correct = pred == p["solution"]
        records.append(dict(id=p["id"], num=p["num_people"], pred=pred,
                            gold=p["solution"], correct=correct,
                            codes=[], text="", num_solutions=len(sols)))
        mark = "✓" if correct else "✗"
        print(f"  [solver] {p['id']} ({p['num_people']}人) {mark}  "
              f"解数={len(sols)}  预测={pred}")
    return records


LABELS = {"pure": "纯思考", "code": "代码辅助", "solver": "约束求解"}


def print_table(columns, puzzles):
    """打印多列准确率对比表。columns = [(mode, records), ...],顺序即列顺序。"""
    accs = {m: sum(r["correct"] for r in recs) / len(recs) for m, recs in columns}
    header = f"{'题号':<8}{'人数':<6}" + "".join(f"{LABELS[m]:<10}" for m, _ in columns)
    print("\n" + "=" * 60)
    print("准确率对比表")
    print("=" * 60)
    print(header)
    print("-" * 60)
    n = len(puzzles)
    for i in range(n):
        row = f"{puzzles[i]['id']:<8}{puzzles[i]['num_people']:<6}"
        for _, recs in columns:
            row += f"{('✓' if recs[i]['correct'] else '✗'):<10}"
        print(row)
    print("-" * 60)
    tail = f"{'准确率':<8}{'':<6}" + "".join(
        f"{accs[m]*100:>6.1f}%   " for m, _ in columns)
    print(tail)
    print("=" * 60)
    for m, recs in columns:
        n_ok = sum(r["correct"] for r in recs)
        print(f"{LABELS[m]:<6} 准确率: {accs[m]*100:5.1f}%  ({n_ok}/{len(recs)})")
    # 若同时有 solver/code 与 pure,报告提升幅度
    baseline = next((m for m in ("pure",) if m in accs), None)
    best = next((m for m in ("solver", "code") if m in accs), None)
    if baseline and best and best != baseline:
        print(f"提升({LABELS[best]} - {LABELS[baseline]}): "
              f"{(accs[best]-accs[baseline])*100:+.1f} 个百分点")


def main():
    global MODEL
    ap = argparse.ArgumentParser(
        description="实验 5-2:对比纯思考 / 代码辅助 / 约束求解 三种模式求解"
                    "「骑士与无赖」逻辑谜题的准确率",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog=__doc__)
    ap.add_argument("--mode", choices=["both", "pure", "code", "solver"],
                    default="both",
                    help="运行模式:both=纯思考+代码辅助(默认);pure=仅纯思考;"
                         "code=仅代码辅助;solver=离线约束求解基线(无需 API)")
    ap.add_argument("--model", default=MODEL,
                    help=f"LLM 模型名(默认 {MODEL};solver 模式忽略)")
    ap.add_argument("--limit", type=int, default=0,
                    help="只跑前 N 题(0=全部)")
    ap.add_argument("--min-people", type=int, default=0,
                    help="只跑居民数 >= 该值的谜题(按难度筛选,0=不限)")
    ap.add_argument("--max-people", type=int, default=0,
                    help="只跑居民数 <= 该值的谜题(按难度筛选,0=不限)")
    ap.add_argument("--puzzles", default="puzzles.json",
                    help="谜题数据集路径(默认 puzzles.json)")
    ap.add_argument("--output", default="last_run.json",
                    help="逐题完整记录的输出路径(默认 last_run.json)")
    args = ap.parse_args()
    MODEL = args.model

    with open(args.puzzles, encoding="utf-8") as f:
        puzzles = json.load(f)
    if args.min_people:
        puzzles = [p for p in puzzles if p["num_people"] >= args.min_people]
    if args.max_people:
        puzzles = [p for p in puzzles if p["num_people"] <= args.max_people]
    if args.limit:
        puzzles = puzzles[:args.limit]
    if not puzzles:
        sys.exit("错误:筛选后没有任何谜题,请放宽 --min-people/--max-people/--limit。")

    # solver 模式完全离线,不需要 API;其余模式需要 OPENAI_API_KEY。
    llm_modes = {"both": ["pure", "code"], "pure": ["pure"],
                 "code": ["code"], "solver": []}[args.mode]
    results = {}

    if args.mode == "solver":
        print(f"离线约束求解基线    题目数:{len(puzzles)}\n")
        print("== 约束求解(solver,离线) ==")
        results["solver"] = run_solver(puzzles)
    else:
        if not (os.environ.get("OPENAI_API_KEY") or os.environ.get("OPENROUTER_API_KEY")):
            sys.exit("错误:pure/code/both 模式需要 OPENAI_API_KEY(或 OPENROUTER_API_KEY 兜底)"
                     "环境变量(可写入 .env)。若只想看离线约束求解基线,请用 --mode solver。")
        client, MODEL = build_client_and_model()  # 延迟导入 openai + OpenRouter 兜底
        print(f"模型:{MODEL}    题目数:{len(puzzles)}    模式:{args.mode}\n")
        for m in llm_modes:
            print(f"== {LABELS[m]}({m}) ==")
            results[m] = run_mode(client, puzzles, m)
            print()

    # ---- 准确率对比表(按 pure -> code -> solver 的固定列序) ----
    columns = [(m, results[m]) for m in ["pure", "code", "solver"] if m in results]
    print_table(columns, puzzles)

    # ---- 展示一题的约束建模代码与求解结果 ----
    code_recs = results.get("code")
    if code_recs:
        sample = next((r for r in code_recs if r["correct"] and r["codes"]), None)
        if sample:
            print("\n" + "=" * 60)
            print(f"示例:{sample['id']} 的约束建模代码(模型生成)")
            print("=" * 60)
            print(sample["codes"][0])
            print("-- 求解 & 最终答案 --")
            print(f"预测={sample['pred']}  真值={sample['gold']}")

    # 保存完整记录,便于复盘
    payload = dict(model=MODEL, mode=args.mode)
    for m, recs in results.items():
        payload[m] = recs
        payload[f"{m}_acc"] = sum(r["correct"] for r in recs) / len(recs)
    with open(args.output, "w", encoding="utf-8") as f:
        json.dump(payload, f, ensure_ascii=False, indent=2)
    print(f"\n完整逐题记录已保存到 {args.output}")


if __name__ == "__main__":
    main()