◉ClassyGitShadowzzh

harness.py

harness.py
"""
四层分层验证 harness。

输入:一条被测 Agent 的运行轨迹(trajectory,schema 见 agent.py),以及对应的任务定义。
输出:四层各自的分数与总评。

四层:
  L1 任务正确性        —— 用 dataset 的 correctness_criteria(规则/判据)核对最终答案。
  L2 工具发现有效性    —— 启发式分析搜索关键词 / 访问网页 / 选库,判断发现是否切题且避开陷阱。
  L3 工具创造质量      —— LLM-as-a-Judge,按 Rubric 给创造出的工具代码打分(错误处理/参数校验/文档)。
  L4 工具复用能力      —— 分析"第二次相似任务"轨迹,是否直接检索已注册工具而非重复搜索创建。
"""

import json
import re
from typing import Optional

from config import Config


# 各层在总评中的权重(若某层 N/A 则在可用层间按比例重新归一)
LAYER_WEIGHTS = {"L1": 0.35, "L2": 0.25, "L3": 0.25, "L4": 0.15}

# 全部四层,供 CLI / 上层选择使用
ALL_LAYERS = ("L1", "L2", "L3", "L4")


# ---------------------------------------------------------------------------
# L1 任务正确性
# ---------------------------------------------------------------------------
def layer1_correctness(task: dict, trajectory: dict) -> dict:
    answer = trajectory.get("final_answer", "") or ""
    crit = task["correctness_criteria"]
    check = crit["check"]
    passed = False
    if check == "regex":
        passed = re.search(crit["pattern"], answer) is not None
    elif check == "contains_any":
        low = answer.lower()
        passed = any(v.lower() in low for v in crit["values"])
    return {
        "score": 1.0 if passed else 0.0,
        "passed": passed,
        "detail": f"判据[{check}] -> {'通过' if passed else '未通过'};{crit['description']}",
    }


# ---------------------------------------------------------------------------
# L2 工具发现有效性
# ---------------------------------------------------------------------------
def _selected_libraries(trajectory: dict):
    return [s["library"] for s in trajectory["steps"] if s["action"] == "select_library"]


def _search_queries(trajectory: dict):
    return [s["query"] for s in trajectory["steps"] if s["action"] == "search"]


def layer2_discovery(task: dict, trajectory: dict) -> dict:
    steps = trajectory["steps"]
    reused = any(s["action"] == "retrieve_tool" for s in steps)
    did_discovery = any(s["action"] in ("search", "select_library", "create_tool") for s in steps)
    if reused and not did_discovery:
        # 本次是复用,没有新的发现活动 —— 该层不适用
        return {"score": None, "detail": "本次直接复用已注册工具,无新发现活动,L2 不适用。"}

    queries = _search_queries(trajectory)
    selected = _selected_libraries(trajectory)
    kws = [k.lower() for k in task.get("discovery_keywords", [])]
    recommended = [l.lower() for l in task["reference_solution"]["libraries"]]
    pit = task.get("known_pitfalls", {})
    bad_libs = [b.lower() for b in (pit.get("deprecated_libraries", []) + pit.get("paid_or_registration_apis", []))]

    # 各项启发式指标
    on_topic = any(any(k in q.lower() for k in kws) for q in queries) if queries else False
    visited_web = any(s["action"] == "read_web" for s in steps)

    def _match(lib, pool):
        lo = lib.lower()
        return any(p.split("(")[0].strip() in lo or lo in p for p in pool)

    selected_recommended = any(_match(l, recommended) for l in selected)
    hit_pitfall = any(_match(l, bad_libs) for l in selected)
    avoided_pitfalls = not hit_pitfall

    score = (
        0.40 * selected_recommended
        + 0.25 * on_topic
        + 0.25 * avoided_pitfalls
        + 0.10 * visited_web
    )
    return {
        "score": round(score, 3),
        "components": {
            "on_topic_search": on_topic,
            "visited_web": visited_web,
            "selected_recommended_lib": selected_recommended,
            "avoided_pitfalls": avoided_pitfalls,
        },
        "selected_libraries": selected,
        "detail": (
            f"搜索切题={on_topic} 访问网页={visited_web} 选中推荐库={selected_recommended} "
            f"避开陷阱={avoided_pitfalls}(选库:{selected})"
        ),
    }


# ---------------------------------------------------------------------------
# L3 工具创造质量(LLM-as-a-Judge,按 Rubric)
# ---------------------------------------------------------------------------
_JUDGE_SYSTEM = (
    "你是一名严格的代码评审专家,负责评估一个自我进化 Agent 自动创造的 Python 工具函数的质量。"
    "请只依据给定的代码本身打分,按下面 4 个维度各打 0-3 分(0=完全没有,1=很弱,2=一般,3=优秀):\n"
    "  error_handling  错误处理:是否用 try/except 处理网络/IO/解析等异常,给出有用信息。\n"
    "  input_validation 参数校验:是否检查入参类型/取值/边界,非法输入是否报错。\n"
    "  documentation   文档完整性:是否有清晰 docstring 说明用途、参数、返回、异常。\n"
    "  robustness      健壮性与契合度:实现是否契合任务目标、是否考虑边界与失败情形。\n"
    "只返回 JSON,形如:"
    '{"error_handling":int,"input_validation":int,"documentation":int,"robustness":int,"comment":"简短中文点评"}'
)


def _parse_judge_json(text: str) -> Optional[dict]:
    try:
        return json.loads(text)
    except Exception:
        m = re.search(r"\{.*\}", text, re.DOTALL)
        if m:
            try:
                return json.loads(m.group(0))
            except Exception:
                return None
    return None


def layer3_tool_quality(task: dict, trajectory: dict, judge_model: Optional[str] = None) -> dict:
    created = trajectory.get("created_tools", [])
    if not created:
        return {"score": None, "detail": "本次轨迹未创造新工具(可能为复用),L3 不适用。"}

    tool = created[0]
    model = Config.map_model(judge_model or Config.JUDGE_MODEL)
    client = Config.get_client()
    user = (
        f"任务目标:{task['goal']}\n\n"
        f"Agent 创造的工具函数 `{tool['name']}` 代码如下:\n```python\n{tool['code']}\n```"
    )
    kwargs = dict(
        model=model,
        temperature=0.0,
        messages=[
            {"role": "system", "content": _JUDGE_SYSTEM},
            {"role": "user", "content": user},
        ],
    )
    try:
        resp = client.chat.completions.create(response_format={"type": "json_object"}, **kwargs)
    except Exception:
        resp = client.chat.completions.create(**kwargs)  # 部分模型不支持 json_object
    raw = resp.choices[0].message.content or ""
    rubric = _parse_judge_json(raw)
    if not rubric:
        return {"score": 0.0, "rubric": None, "judge_text": raw, "detail": "judge 输出无法解析为 JSON。"}

    dims = ["error_handling", "input_validation", "documentation", "robustness"]
    total = sum(int(rubric.get(d, 0)) for d in dims)
    score = round(total / (3 * len(dims)), 3)
    return {
        "score": score,
        "rubric": rubric,
        "judge_text": raw,
        "tool_name": tool["name"],
        "detail": (
            f"Rubric 4 维合计 {total}/12 -> 归一 {score};"
            f"点评:{rubric.get('comment', '')}"
        ),
    }


# ---------------------------------------------------------------------------
# L4 工具复用能力(分析第二次相似任务的轨迹)
# ---------------------------------------------------------------------------
def layer4_reuse(task: dict, variant_trajectory: dict) -> dict:
    if variant_trajectory is None:
        return {"score": None, "detail": "未提供第二次相似任务轨迹,L4 未测。"}
    steps = variant_trajectory["steps"]
    retrieved = any(
        s["action"] == "retrieve_tool" and s.get("name") == task["tool_name"] for s in steps
    )
    re_searched = any(s["action"] == "search" for s in steps)
    re_created = any(s["action"] == "create_tool" for s in steps)

    if retrieved and not re_searched and not re_created:
        score, verdict = 1.0, "直接检索并复用已注册工具(未重复搜索/创建)"
    elif retrieved and (re_searched or re_created):
        score, verdict = 0.5, "检索到工具但仍有重复搜索/创建"
    else:
        score, verdict = 0.0, "未复用,重复了搜索与工具创建"
    return {
        "score": score,
        "retrieved_from_registry": retrieved,
        "re_searched": re_searched,
        "re_created": re_created,
        "detail": verdict,
    }


# ---------------------------------------------------------------------------
# 汇总
# ---------------------------------------------------------------------------
def aggregate(layers: dict) -> dict:
    avail = {k: v["score"] for k, v in layers.items() if v.get("score") is not None}
    if not avail:
        return {"overall": None, "used_layers": []}
    wsum = sum(LAYER_WEIGHTS[k] for k in avail)
    overall = sum(LAYER_WEIGHTS[k] * s for k, s in avail.items()) / wsum
    return {"overall": round(overall, 3), "used_layers": list(avail)}


class FourLayerEvaluator:
    """把四层封装到一起。variant_trajectory 用于 L4。

    layers 指定实际运行哪些层(默认四层全跑)。只有 L3 需要联网调用 LLM,
    因此离线场景可传 layers=("L1","L2","L4") 跳过 L3——未选中的层记 N/A,不参与总评。"""

    def __init__(self, judge_model: Optional[str] = None, layers=ALL_LAYERS):
        self.judge_model = judge_model or Config.JUDGE_MODEL
        self.layers = tuple(layers)

    def evaluate(self, task: dict, trajectory: dict, variant_trajectory: Optional[dict] = None) -> dict:
        skipped = {"score": None, "detail": "(本次未选择该层,记 N/A)"}
        layers = {
            "L1": layer1_correctness(task, trajectory) if "L1" in self.layers else dict(skipped),
            "L2": layer2_discovery(task, trajectory) if "L2" in self.layers else dict(skipped),
            "L3": (
                layer3_tool_quality(task, trajectory, self.judge_model)
                if "L3" in self.layers else dict(skipped)
            ),
            "L4": layer4_reuse(task, variant_trajectory) if "L4" in self.layers else dict(skipped),
        }
        return {
            "task_id": task["id"],
            "domain": task["domain"],
            "profile": trajectory.get("profile"),
            "layers": layers,
            "summary": aggregate(layers),
        }