ClassyGitShadowzzh

pipeline.py

pipeline.py
"""TTS 质量评估流水线的核心步骤。

一条评估链路:
  合成(OpenAI TTS) -> 时长探测(ffprobe) -> 回译(Whisper) -> 计算 CER/字准确率
      -> LLM Rubric 打分(gpt-5.6-luna) [可选: Gemini 音频评审 gemini-3.5-flash]

说明:TTS 合成与 Whisper 回译必须走 OpenAI 直连(OpenRouter 不提供音频/转写);
仅 LLM Rubric 的 chat 评审支持 OpenRouter 回退——gpt-5.x 直连需组织实名认证,
故只要有 OPENROUTER_API_KEY 就优先经 OpenRouter 调评审模型(见 get_judge_client_and_model)。

所有对外函数都做了健壮性处理:单条失败抛出带上下文的异常,由 demo.py 捕获后
在汇总表里记为失败,而不会中断整表。
"""

import base64
import json
import os
import re
import shutil
import subprocess
from dataclasses import dataclass, field
from typing import Optional

from openai import OpenAI

import config

# ---------------------------------------------------------------------------
# 客户端(带自动重试,缓解偶发的网络抖动)。
# ---------------------------------------------------------------------------
_client: Optional[OpenAI] = None


def get_client() -> OpenAI:
    """OpenAI 直连 client:用于 TTS 合成与 Whisper 回译(这两项不能走 OpenRouter)。"""
    global _client
    if _client is None:
        key = os.environ.get("OPENAI_API_KEY", "").strip()
        if not key:
            raise RuntimeError(
                "缺少 OPENAI_API_KEY(TTS 合成 / Whisper 回译需 OpenAI 直连)。"
                "请 `export OPENAI_API_KEY=sk-...` 或写入 .env。"
            )
        _client = OpenAI(api_key=key, max_retries=5, timeout=60.0)
    return _client


# ---------------------------------------------------------------------------
# LLM Rubric 评审客户端:支持 OpenRouter 回退。
# gpt-5.x 直连 OpenAI 需组织实名认证,只要有 OPENROUTER_API_KEY 就优先走 OpenRouter。
# 注意:仅 chat 评审可回退;TTS / Whisper 仍需 OpenAI 直连(见 get_client)。
# ---------------------------------------------------------------------------
OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1"
_judge_client: Optional[OpenAI] = None
_judge_client_kind: str = ""


def _to_openrouter_model(model: str) -> str:
    """把模型名映射成 OpenRouter id:含 '/' 视为原生 id;gpt-* -> openai/*;
    claude-* -> anthropic/claude-opus-4.8;其余回退到 openai/gpt-5.6-luna。"""
    if "/" in model:
        return model
    if model.startswith("gpt-"):
        return "openai/" + model
    if model.startswith("claude-"):
        return "anthropic/claude-opus-4.8"
    return "openai/gpt-5.6-luna"


def get_judge_client_and_model(model: str):
    """构造 LLM 评审用的 client 并返回 (client, 实际模型名)。

    回退:gpt-5.x 且有 OPENROUTER_API_KEY -> 优先 OpenRouter;否则有 OPENAI_API_KEY ->
    直连;否则有 OPENROUTER_API_KEY -> OpenRouter(模型名映射);皆无 -> 清晰报错。
    """
    global _judge_client, _judge_client_kind
    primary = os.environ.get("OPENAI_API_KEY", "").strip()
    orkey = os.environ.get("OPENROUTER_API_KEY", "").strip()
    prefer_or = bool(orkey) and model.startswith("gpt-5")

    if not prefer_or and primary:
        if _judge_client_kind != "openai":
            _judge_client = OpenAI(api_key=primary, max_retries=5, timeout=60.0)
            _judge_client_kind = "openai"
        return _judge_client, model
    if orkey:
        if _judge_client_kind != "openrouter":
            _judge_client = OpenAI(base_url=OPENROUTER_BASE_URL, api_key=orkey,
                                   max_retries=5, timeout=60.0)
            _judge_client_kind = "openrouter"
        return _judge_client, _to_openrouter_model(model)
    if primary:
        if _judge_client_kind != "openai":
            _judge_client = OpenAI(api_key=primary, max_retries=5, timeout=60.0)
            _judge_client_kind = "openai"
        return _judge_client, model
    raise RuntimeError(
        "缺少 OPENAI_API_KEY / OPENROUTER_API_KEY,无法运行 LLM Rubric 评审。"
    )


# ---------------------------------------------------------------------------
# 1) TTS 合成(多 provider 分发)
# ---------------------------------------------------------------------------
def synthesize(cfg: config.TTSConfig, text: str, out_path: str) -> None:
    """按 cfg.provider 分发到对应服务商合成语音,写入 out_path(mp3)。失败抛异常。

    OpenAI 走官方 SDK;其余服务商按各家公开 REST 接口用内置 urllib 调用,
    不引入额外依赖。缺少对应 key 时抛出带上下文的异常,由上层记为该行失败。
    """
    fn = _SYNTH_DISPATCH.get(cfg.provider)
    if fn is None:
        raise RuntimeError(
            f"未知 provider: {cfg.provider!r}(可选:{', '.join(_SYNTH_DISPATCH)})"
        )
    audio = fn(cfg, text)
    if not audio:
        raise RuntimeError(f"{cfg.provider} TTS 返回空音频")
    with open(out_path, "wb") as f:
        f.write(audio)


def _require_env(name: str) -> str:
    # 走 config.env_get 以支持环境变量别名(如 Fish 的 FISH_API_KEY / FISHAUDIO_API_KEY)。
    val = config.env_get(name)
    if not val:
        raise RuntimeError(f"缺少环境变量 {name},无法用该 provider 合成。")
    return val


def _http_post(url: str, body: dict, headers: dict, timeout: float = 90.0) -> bytes:
    """POST JSON,返回原始响应字节。非 2xx 抛出带响应体片段的异常。"""
    import urllib.error
    import urllib.request
    req = urllib.request.Request(
        url, data=json.dumps(body).encode(),
        headers={"Content-Type": "application/json", **headers}, method="POST",
    )
    try:
        with urllib.request.urlopen(req, timeout=timeout) as r:
            return r.read()
    except urllib.error.HTTPError as e:
        detail = e.read().decode("utf-8", "replace")[:300]
        raise RuntimeError(f"HTTP {e.code}: {detail}") from None


def _synth_openai(cfg: config.TTSConfig, text: str) -> bytes:
    kwargs = dict(model=cfg.model, voice=cfg.voice, input=text)
    if cfg.supports_speed() and abs(cfg.speed - 1.0) > 1e-6:
        kwargs["speed"] = cfg.speed
    return get_client().audio.speech.create(**kwargs).content


def _synth_elevenlabs(cfg: config.TTSConfig, text: str) -> bytes:
    key = _require_env("ELEVENLABS_API_KEY")
    voice = cfg.voice or "21m00Tcm4TlvDq8ikWAM"
    url = (f"https://api.elevenlabs.io/v1/text-to-speech/{voice}"
           f"?output_format=mp3_44100_128")
    body = {"text": text, "model_id": cfg.model or "eleven_multilingual_v2"}
    # ElevenLabs 返回原始 mp3 字节。
    return _http_post(url, body, {"xi-api-key": key, "Accept": "audio/mpeg"})


def _synth_fishaudio(cfg: config.TTSConfig, text: str) -> bytes:
    key = _require_env("FISH_API_KEY")
    body = {"text": text, "format": "mp3"}
    if cfg.voice:
        body["reference_id"] = cfg.voice
    # Fish Audio /v1/tts 接受 JSON,直接返回音频字节。
    return _http_post("https://api.fish.audio/v1/tts", body,
                      {"Authorization": f"Bearer {key}"})


def _synth_minimax(cfg: config.TTSConfig, text: str) -> bytes:
    key = _require_env("MINIMAX_API_KEY")
    group = _require_env("MINIMAX_GROUP_ID")
    url = f"https://api.minimax.chat/v1/t2a_v2?GroupId={group}"
    body = {
        "model": cfg.model or "speech-01-turbo",
        "text": text,
        "stream": False,
        "voice_setting": {"voice_id": cfg.voice, "speed": cfg.speed},
        "audio_setting": {"format": "mp3", "sample_rate": 32000},
    }
    raw = _http_post(url, body, {"Authorization": f"Bearer {key}"})
    data = json.loads(raw)
    # 返回 JSON,音频为 data.audio(hex 编码)。
    hexstr = (data.get("data") or {}).get("audio")
    if not hexstr:
        err = data.get("base_resp", {})
        raise RuntimeError(f"Minimax 无音频返回:{err or data}")
    return bytes.fromhex(hexstr)


def _synth_doubao(cfg: config.TTSConfig, text: str) -> bytes:
    import uuid
    appid = _require_env("DOUBAO_APP_ID")
    token = _require_env("DOUBAO_ACCESS_TOKEN")
    body = {
        "app": {"appid": appid, "token": token,
                "cluster": cfg.model or "volcano_tts"},
        "user": {"uid": "tts-quality-eval"},
        "audio": {"voice_type": cfg.voice, "encoding": "mp3",
                  "speed_ratio": cfg.speed},
        "request": {"reqid": str(uuid.uuid4()), "text": text, "operation": "query"},
    }
    # 火山引擎鉴权头是特殊的 'Bearer;{token}' 形式;音频为 base64 编码的 data 字段。
    raw = _http_post("https://openspeech.bytedance.com/api/v1/tts", body,
                     {"Authorization": f"Bearer;{token}"})
    data = json.loads(raw)
    b64 = data.get("data")
    if not b64:
        raise RuntimeError(f"豆包无音频返回:code={data.get('code')} "
                           f"message={data.get('message')}")
    return base64.b64decode(b64)


_SYNTH_DISPATCH = {
    "openai": _synth_openai,
    "elevenlabs": _synth_elevenlabs,
    "fishaudio": _synth_fishaudio,
    "minimax": _synth_minimax,
    "doubao": _synth_doubao,
}


# ---------------------------------------------------------------------------
# 2) 时长探测(ffprobe)
# ---------------------------------------------------------------------------
def probe_duration(path: str) -> float:
    """返回音频时长(秒)。ffprobe 缺失或出错时抛异常。"""
    if shutil.which("ffprobe") is None:
        raise RuntimeError("未找到 ffprobe,请安装 ffmpeg(macOS: brew install ffmpeg)。")
    proc = subprocess.run(
        ["ffprobe", "-v", "error", "-show_entries", "format=duration",
         "-of", "default=noprint_wrappers=1:nokey=1", path],
        capture_output=True, text=True,
    )
    if proc.returncode != 0:
        raise RuntimeError(f"ffprobe 失败: {proc.stderr.strip()}")
    out = proc.stdout.strip()
    try:
        return float(out)
    except ValueError:
        raise RuntimeError(f"ffprobe 输出无法解析为时长: {out!r}")


# ---------------------------------------------------------------------------
# 3) 回译(Whisper 转写)
# ---------------------------------------------------------------------------
# 用简体中文提示语引导 Whisper 输出简体,避免它偶尔返回繁体导致 CER 被字形差异
# 虚高(那是转写脚本选择问题,不是 TTS 发音错误)。
_ZH_PROMPT = "以下是普通话简体中文的句子。"


def transcribe(path: str) -> str:
    with open(path, "rb") as f:
        tr = get_client().audio.transcriptions.create(
            model=config.WHISPER_MODEL, file=f, language="zh", prompt=_ZH_PROMPT,
        )
    return tr.text or ""


# ---------------------------------------------------------------------------
# 4) 文本归一化 + 字错误率(中文用字级 CER,等价于书中所说 WER 的可懂度维度)
# ---------------------------------------------------------------------------
def normalize(text: str) -> str:
    """去掉标点/空白,只保留 CJK / 字母 / 数字,并小写,便于逐字比较。"""
    text = text.lower()
    return "".join(ch for ch in text if ch.isalnum())


def _edit_distance(a: str, b: str) -> int:
    """Levenshtein 距离(字符级)。"""
    if a == b:
        return 0
    if not a:
        return len(b)
    if not b:
        return len(a)
    prev = list(range(len(b) + 1))
    for i, ca in enumerate(a, 1):
        cur = [i]
        for j, cb in enumerate(b, 1):
            cur.append(min(
                prev[j] + 1,        # 删除
                cur[j - 1] + 1,     # 插入
                prev[j - 1] + (ca != cb),  # 替换
            ))
        prev = cur
    return prev[-1]


@dataclass
class ErrorRate:
    cer: float          # 字错误率 = 编辑距离 / 参考字数
    accuracy: float     # 字准确率 = 1 - cer(下限 0)
    edits: int
    ref_len: int


def char_error_rate(reference: str, hypothesis: str) -> ErrorRate:
    ref = normalize(reference)
    hyp = normalize(hypothesis)
    if not ref:
        return ErrorRate(0.0, 1.0, 0, 0)
    dist = _edit_distance(ref, hyp)
    cer = dist / len(ref)
    return ErrorRate(cer=cer, accuracy=max(0.0, 1.0 - cer), edits=dist, ref_len=len(ref))


# ---------------------------------------------------------------------------
# 5) LLM Rubric 评审(默认,OpenAI 闭环)
# ---------------------------------------------------------------------------
RUBRIC_DIMENSIONS = ["清晰度", "自然度", "停顿节奏", "整体"]

# 维度说明(供 --dump-rubric 离线打印,也是评审 prompt 的依据)。括号内标注与书中
# 四维度(准确性 / 自然度 / 情感表达 / 音色一致性)的对应关系。
RUBRIC_DESCRIPTIONS = {
    "清晰度": "转写与原文是否高度一致,漏字/错字/多字越多分越低(对应书中「准确性」)。",
    "自然度": "语速是否接近自然朗读(中文约 4-6 字/秒),过快>7 或过慢<3 都不自然。",
    "停顿节奏": "结合语速与文本长度判断停顿/节奏是否合理,过快通常意味吞字、节奏差。",
    "整体": "综合以上给出的总体印象分。",
}
# 说明:默认(回译)评审看不到音频,无法覆盖书中「情感表达 / 音色一致性」;这两维需
# 多模态直接听音频,用 --gemini 复现(音色一致性还需参考语音,本 demo 未提供)。

_JUDGE_SYSTEM = """你是严格的 TTS(文本转语音)质量评审专家。
你将拿到:原始参考文本、该文本的期望情感、由 Whisper 对合成语音回译得到的转写文本,
以及从音频客观测得的时长、语速(字/秒)和字错误率(CER)。
请据此对合成语音质量按 Rubric 逐维度打分(1-5 的整数,5 最好):

- 清晰度:转写与原文是否高度一致(漏字/错字/多字越多分越低;CER 越高分越低)。
- 自然度:语速是否接近自然朗读(中文自然朗读约 4-6 字/秒;过快>7 或过慢<3 都不自然)。
- 停顿节奏:结合语速与文本长度,判断停顿/节奏是否合理(过快通常意味着吞字、节奏差)。
- 整体:综合以上给出的总体印象分。

注意:你看不到音频本身,只能基于以上可测特征做保守、可解释的判断。
只输出 JSON,格式:
{"清晰度": {"score": int, "reason": str},
 "自然度": {"score": int, "reason": str},
 "停顿节奏": {"score": int, "reason": str},
 "整体": {"score": int, "reason": str}}
reason 用一句简短中文说明。"""


@dataclass
class RubricResult:
    scores: dict            # 维度 -> int
    reasons: dict           # 维度 -> str
    raw: str = ""


def judge_rubric(reference: str, emotion: str, hypothesis: str,
                 duration: float, cer: float, model: Optional[str] = None) -> RubricResult:
    """用评审模型(默认 gpt-5.6-luna)按 Rubric 打分。返回结构化分数 + 点评。

    评审 chat 调用支持 OpenRouter 回退(见 get_judge_client_and_model)。"""
    chars = len(normalize(reference))
    speed = chars / duration if duration > 0 else 0.0
    user = (
        f"原始参考文本:{reference}\n"
        f"期望情感:{emotion}\n"
        f"Whisper 回译文本:{hypothesis}\n"
        f"音频时长:{duration:.2f} 秒\n"
        f"语速:{speed:.2f} 字/秒(参考文本 {chars} 个有效字符)\n"
        f"字错误率 CER:{cer:.3f}\n"
    )
    judge_client, judge_model = get_judge_client_and_model(model or config.JUDGE_MODEL)
    resp = judge_client.chat.completions.create(
        model=judge_model,
        messages=[{"role": "system", "content": _JUDGE_SYSTEM},
                  {"role": "user", "content": user}],
        temperature=0.0,
        response_format={"type": "json_object"},
    )
    raw = resp.choices[0].message.content or "{}"
    data = json.loads(raw)
    scores, reasons = {}, {}
    for dim in RUBRIC_DIMENSIONS:
        item = data.get(dim, {})
        if isinstance(item, dict):
            scores[dim] = int(item.get("score") or 0)   # score 缺失或为 null 时按 0 分
            reasons[dim] = str(item.get("reason", "")).strip()
        else:  # 兼容模型直接返回数字(null 按 0 分)
            scores[dim] = int(item or 0)
            reasons[dim] = ""
    return RubricResult(scores=scores, reasons=reasons, raw=raw)


# ---------------------------------------------------------------------------
# 6) 可选:Gemini 多模态音频评审(书中方案)。用 REST,避免额外 SDK 依赖。
# ---------------------------------------------------------------------------
def _resolve_gemini_model(api_key: str) -> str:
    """探测当前可用的 Gemini 模型,避免默认名过期。"""
    import urllib.request
    url = f"https://generativelanguage.googleapis.com/v1beta/models?key={api_key}"
    try:
        with urllib.request.urlopen(url, timeout=20) as r:
            data = json.loads(r.read())
        names = [m["name"].split("/")[-1] for m in data.get("models", [])
                 if "generateContent" in m.get("supportedGenerationMethods", [])]
        # 优先默认的 gemini-3.5-flash(已验证支持音频输入),再退到 pro / 旧 flash 系列。
        for want in (config.GEMINI_MODEL_DEFAULT, "gemini-3.5-flash",
                     "gemini-2.5-pro", "gemini-2.5-flash", "gemini-flash-latest"):
            if want in names:
                return want
        # 退而求其次:任意非 tts/image 的可用模型
        for n in names:
            if "tts" not in n and "image" not in n and "embedding" not in n:
                return n
    except Exception:
        pass
    return config.GEMINI_MODEL_DEFAULT


def judge_gemini_audio(reference: str, emotion: str, audio_path: str) -> RubricResult:
    """把合成音频 + 原文 + Rubric 一起交给 Gemini 多模态直接「听」并打分。

    需要 GEMINI_API_KEY。默认关闭;--gemini 开启。失败抛异常由上层记为失败。
    """
    import urllib.request
    key = os.environ.get("GEMINI_API_KEY", "").strip()
    if not key:
        raise RuntimeError("缺少 GEMINI_API_KEY,无法使用 Gemini 音频评审。")
    model = _resolve_gemini_model(key)
    with open(audio_path, "rb") as f:
        audio_b64 = base64.b64encode(f.read()).decode()
    prompt = (
        "你是 TTS 质量评审专家。请直接聆听下面这段合成语音,对照原始文本与期望情感,"
        "按 1-5 分为四个维度打分并给出简短理由,只输出 JSON:"
        '{"清晰度":{"score":int,"reason":str},"自然度":{"score":int,"reason":str},'
        '"停顿节奏":{"score":int,"reason":str},"整体":{"score":int,"reason":str}}\n'
        f"原始文本:{reference}\n期望情感:{emotion}"
    )
    body = {
        "contents": [{"parts": [
            {"text": prompt},
            {"inline_data": {"mime_type": "audio/mp3", "data": audio_b64}},
        ]}],
        "generationConfig": {"temperature": 0.0, "responseMimeType": "application/json"},
    }
    url = (f"https://generativelanguage.googleapis.com/v1beta/models/"
           f"{model}:generateContent?key={key}")
    req = urllib.request.Request(
        url, data=json.dumps(body).encode(),
        headers={"Content-Type": "application/json"}, method="POST",
    )
    with urllib.request.urlopen(req, timeout=90) as r:
        data = json.loads(r.read())
    # Gemini 在安全拦截时不返回 candidates(或 candidate 无 content/parts),
    # 防御式取值并给出带 promptFeedback 的清晰错误,交由上层记为该条失败。
    candidates = data.get("candidates") or []
    parts = []
    if candidates:
        parts = (candidates[0].get("content") or {}).get("parts") or []
    if not parts or not parts[0].get("text"):
        raise RuntimeError(f"Gemini 未返回评审文本:{data.get('promptFeedback') or data}")
    text = parts[0]["text"]
    parsed = json.loads(text)
    scores, reasons = {}, {}
    # 评审 JSON 的 score 字段缺失或为 null 时按 0 分处理,与 judge_rubric 一致
    for dim in RUBRIC_DIMENSIONS:
        item = parsed.get(dim, {})
        scores[dim] = int(item.get("score") or 0) if isinstance(item, dict) else int(item or 0)
        reasons[dim] = str(item.get("reason", "")).strip() if isinstance(item, dict) else ""
    return RubricResult(scores=scores, reasons=reasons, raw=text)