"""TTS 质量评估流水线的核心步骤。
一条评估链路:
合成(OpenAI TTS) -> 时长探测(ffprobe) -> 回译(Whisper) -> 计算 CER/字准确率
-> LLM Rubric 打分(gpt-5.6-luna) [可选: Gemini 音频评审 gemini-3.5-flash]
说明:TTS 合成与 Whisper 回译必须走 OpenAI 直连(OpenRouter 不提供音频/转写);
仅 LLM Rubric 的 chat 评审支持 OpenRouter 回退——gpt-5.x 直连需组织实名认证,
故只要有 OPENROUTER_API_KEY 就优先经 OpenRouter 调评审模型(见 get_judge_client_and_model)。
所有对外函数都做了健壮性处理:单条失败抛出带上下文的异常,由 demo.py 捕获后
在汇总表里记为失败,而不会中断整表。
"""
import base64
import json
import os
import re
import shutil
import subprocess
from dataclasses import dataclass, field
from typing import Optional
from openai import OpenAI
import config
# ---------------------------------------------------------------------------
# 客户端(带自动重试,缓解偶发的网络抖动)。
# ---------------------------------------------------------------------------
_client: Optional[OpenAI] = None
def get_client() -> OpenAI:
"""OpenAI 直连 client:用于 TTS 合成与 Whisper 回译(这两项不能走 OpenRouter)。"""
global _client
if _client is None:
key = os.environ.get("OPENAI_API_KEY", "").strip()
if not key:
raise RuntimeError(
"缺少 OPENAI_API_KEY(TTS 合成 / Whisper 回译需 OpenAI 直连)。"
"请 `export OPENAI_API_KEY=sk-...` 或写入 .env。"
)
_client = OpenAI(api_key=key, max_retries=5, timeout=60.0)
return _client
# ---------------------------------------------------------------------------
# LLM Rubric 评审客户端:支持 OpenRouter 回退。
# gpt-5.x 直连 OpenAI 需组织实名认证,只要有 OPENROUTER_API_KEY 就优先走 OpenRouter。
# 注意:仅 chat 评审可回退;TTS / Whisper 仍需 OpenAI 直连(见 get_client)。
# ---------------------------------------------------------------------------
OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1"
_judge_client: Optional[OpenAI] = None
_judge_client_kind: str = ""
def _to_openrouter_model(model: str) -> str:
"""把模型名映射成 OpenRouter id:含 '/' 视为原生 id;gpt-* -> openai/*;
claude-* -> anthropic/claude-opus-4.8;其余回退到 openai/gpt-5.6-luna。"""
if "/" in model:
return model
if model.startswith("gpt-"):
return "openai/" + model
if model.startswith("claude-"):
return "anthropic/claude-opus-4.8"
return "openai/gpt-5.6-luna"
def get_judge_client_and_model(model: str):
"""构造 LLM 评审用的 client 并返回 (client, 实际模型名)。
回退:gpt-5.x 且有 OPENROUTER_API_KEY -> 优先 OpenRouter;否则有 OPENAI_API_KEY ->
直连;否则有 OPENROUTER_API_KEY -> OpenRouter(模型名映射);皆无 -> 清晰报错。
"""
global _judge_client, _judge_client_kind
primary = os.environ.get("OPENAI_API_KEY", "").strip()
orkey = os.environ.get("OPENROUTER_API_KEY", "").strip()
prefer_or = bool(orkey) and model.startswith("gpt-5")
if not prefer_or and primary:
if _judge_client_kind != "openai":
_judge_client = OpenAI(api_key=primary, max_retries=5, timeout=60.0)
_judge_client_kind = "openai"
return _judge_client, model
if orkey:
if _judge_client_kind != "openrouter":
_judge_client = OpenAI(base_url=OPENROUTER_BASE_URL, api_key=orkey,
max_retries=5, timeout=60.0)
_judge_client_kind = "openrouter"
return _judge_client, _to_openrouter_model(model)
if primary:
if _judge_client_kind != "openai":
_judge_client = OpenAI(api_key=primary, max_retries=5, timeout=60.0)
_judge_client_kind = "openai"
return _judge_client, model
raise RuntimeError(
"缺少 OPENAI_API_KEY / OPENROUTER_API_KEY,无法运行 LLM Rubric 评审。"
)
# ---------------------------------------------------------------------------
# 1) TTS 合成(多 provider 分发)
# ---------------------------------------------------------------------------
def synthesize(cfg: config.TTSConfig, text: str, out_path: str) -> None:
"""按 cfg.provider 分发到对应服务商合成语音,写入 out_path(mp3)。失败抛异常。
OpenAI 走官方 SDK;其余服务商按各家公开 REST 接口用内置 urllib 调用,
不引入额外依赖。缺少对应 key 时抛出带上下文的异常,由上层记为该行失败。
"""
fn = _SYNTH_DISPATCH.get(cfg.provider)
if fn is None:
raise RuntimeError(
f"未知 provider: {cfg.provider!r}(可选:{', '.join(_SYNTH_DISPATCH)})"
)
audio = fn(cfg, text)
if not audio:
raise RuntimeError(f"{cfg.provider} TTS 返回空音频")
with open(out_path, "wb") as f:
f.write(audio)
def _require_env(name: str) -> str:
# 走 config.env_get 以支持环境变量别名(如 Fish 的 FISH_API_KEY / FISHAUDIO_API_KEY)。
val = config.env_get(name)
if not val:
raise RuntimeError(f"缺少环境变量 {name},无法用该 provider 合成。")
return val
def _http_post(url: str, body: dict, headers: dict, timeout: float = 90.0) -> bytes:
"""POST JSON,返回原始响应字节。非 2xx 抛出带响应体片段的异常。"""
import urllib.error
import urllib.request
req = urllib.request.Request(
url, data=json.dumps(body).encode(),
headers={"Content-Type": "application/json", **headers}, method="POST",
)
try:
with urllib.request.urlopen(req, timeout=timeout) as r:
return r.read()
except urllib.error.HTTPError as e:
detail = e.read().decode("utf-8", "replace")[:300]
raise RuntimeError(f"HTTP {e.code}: {detail}") from None
def _synth_openai(cfg: config.TTSConfig, text: str) -> bytes:
kwargs = dict(model=cfg.model, voice=cfg.voice, input=text)
if cfg.supports_speed() and abs(cfg.speed - 1.0) > 1e-6:
kwargs["speed"] = cfg.speed
return get_client().audio.speech.create(**kwargs).content
def _synth_elevenlabs(cfg: config.TTSConfig, text: str) -> bytes:
key = _require_env("ELEVENLABS_API_KEY")
voice = cfg.voice or "21m00Tcm4TlvDq8ikWAM"
url = (f"https://api.elevenlabs.io/v1/text-to-speech/{voice}"
f"?output_format=mp3_44100_128")
body = {"text": text, "model_id": cfg.model or "eleven_multilingual_v2"}
# ElevenLabs 返回原始 mp3 字节。
return _http_post(url, body, {"xi-api-key": key, "Accept": "audio/mpeg"})
def _synth_fishaudio(cfg: config.TTSConfig, text: str) -> bytes:
key = _require_env("FISH_API_KEY")
body = {"text": text, "format": "mp3"}
if cfg.voice:
body["reference_id"] = cfg.voice
# Fish Audio /v1/tts 接受 JSON,直接返回音频字节。
return _http_post("https://api.fish.audio/v1/tts", body,
{"Authorization": f"Bearer {key}"})
def _synth_minimax(cfg: config.TTSConfig, text: str) -> bytes:
key = _require_env("MINIMAX_API_KEY")
group = _require_env("MINIMAX_GROUP_ID")
url = f"https://api.minimax.chat/v1/t2a_v2?GroupId={group}"
body = {
"model": cfg.model or "speech-01-turbo",
"text": text,
"stream": False,
"voice_setting": {"voice_id": cfg.voice, "speed": cfg.speed},
"audio_setting": {"format": "mp3", "sample_rate": 32000},
}
raw = _http_post(url, body, {"Authorization": f"Bearer {key}"})
data = json.loads(raw)
# 返回 JSON,音频为 data.audio(hex 编码)。
hexstr = (data.get("data") or {}).get("audio")
if not hexstr:
err = data.get("base_resp", {})
raise RuntimeError(f"Minimax 无音频返回:{err or data}")
return bytes.fromhex(hexstr)
def _synth_doubao(cfg: config.TTSConfig, text: str) -> bytes:
import uuid
appid = _require_env("DOUBAO_APP_ID")
token = _require_env("DOUBAO_ACCESS_TOKEN")
body = {
"app": {"appid": appid, "token": token,
"cluster": cfg.model or "volcano_tts"},
"user": {"uid": "tts-quality-eval"},
"audio": {"voice_type": cfg.voice, "encoding": "mp3",
"speed_ratio": cfg.speed},
"request": {"reqid": str(uuid.uuid4()), "text": text, "operation": "query"},
}
# 火山引擎鉴权头是特殊的 'Bearer;{token}' 形式;音频为 base64 编码的 data 字段。
raw = _http_post("https://openspeech.bytedance.com/api/v1/tts", body,
{"Authorization": f"Bearer;{token}"})
data = json.loads(raw)
b64 = data.get("data")
if not b64:
raise RuntimeError(f"豆包无音频返回:code={data.get('code')} "
f"message={data.get('message')}")
return base64.b64decode(b64)
_SYNTH_DISPATCH = {
"openai": _synth_openai,
"elevenlabs": _synth_elevenlabs,
"fishaudio": _synth_fishaudio,
"minimax": _synth_minimax,
"doubao": _synth_doubao,
}
# ---------------------------------------------------------------------------
# 2) 时长探测(ffprobe)
# ---------------------------------------------------------------------------
def probe_duration(path: str) -> float:
"""返回音频时长(秒)。ffprobe 缺失或出错时抛异常。"""
if shutil.which("ffprobe") is None:
raise RuntimeError("未找到 ffprobe,请安装 ffmpeg(macOS: brew install ffmpeg)。")
proc = subprocess.run(
["ffprobe", "-v", "error", "-show_entries", "format=duration",
"-of", "default=noprint_wrappers=1:nokey=1", path],
capture_output=True, text=True,
)
if proc.returncode != 0:
raise RuntimeError(f"ffprobe 失败: {proc.stderr.strip()}")
out = proc.stdout.strip()
try:
return float(out)
except ValueError:
raise RuntimeError(f"ffprobe 输出无法解析为时长: {out!r}")
# ---------------------------------------------------------------------------
# 3) 回译(Whisper 转写)
# ---------------------------------------------------------------------------
# 用简体中文提示语引导 Whisper 输出简体,避免它偶尔返回繁体导致 CER 被字形差异
# 虚高(那是转写脚本选择问题,不是 TTS 发音错误)。
_ZH_PROMPT = "以下是普通话简体中文的句子。"
def transcribe(path: str) -> str:
with open(path, "rb") as f:
tr = get_client().audio.transcriptions.create(
model=config.WHISPER_MODEL, file=f, language="zh", prompt=_ZH_PROMPT,
)
return tr.text or ""
# ---------------------------------------------------------------------------
# 4) 文本归一化 + 字错误率(中文用字级 CER,等价于书中所说 WER 的可懂度维度)
# ---------------------------------------------------------------------------
def normalize(text: str) -> str:
"""去掉标点/空白,只保留 CJK / 字母 / 数字,并小写,便于逐字比较。"""
text = text.lower()
return "".join(ch for ch in text if ch.isalnum())
def _edit_distance(a: str, b: str) -> int:
"""Levenshtein 距离(字符级)。"""
if a == b:
return 0
if not a:
return len(b)
if not b:
return len(a)
prev = list(range(len(b) + 1))
for i, ca in enumerate(a, 1):
cur = [i]
for j, cb in enumerate(b, 1):
cur.append(min(
prev[j] + 1, # 删除
cur[j - 1] + 1, # 插入
prev[j - 1] + (ca != cb), # 替换
))
prev = cur
return prev[-1]
@dataclass
class ErrorRate:
cer: float # 字错误率 = 编辑距离 / 参考字数
accuracy: float # 字准确率 = 1 - cer(下限 0)
edits: int
ref_len: int
def char_error_rate(reference: str, hypothesis: str) -> ErrorRate:
ref = normalize(reference)
hyp = normalize(hypothesis)
if not ref:
return ErrorRate(0.0, 1.0, 0, 0)
dist = _edit_distance(ref, hyp)
cer = dist / len(ref)
return ErrorRate(cer=cer, accuracy=max(0.0, 1.0 - cer), edits=dist, ref_len=len(ref))
# ---------------------------------------------------------------------------
# 5) LLM Rubric 评审(默认,OpenAI 闭环)
# ---------------------------------------------------------------------------
RUBRIC_DIMENSIONS = ["清晰度", "自然度", "停顿节奏", "整体"]
# 维度说明(供 --dump-rubric 离线打印,也是评审 prompt 的依据)。括号内标注与书中
# 四维度(准确性 / 自然度 / 情感表达 / 音色一致性)的对应关系。
RUBRIC_DESCRIPTIONS = {
"清晰度": "转写与原文是否高度一致,漏字/错字/多字越多分越低(对应书中「准确性」)。",
"自然度": "语速是否接近自然朗读(中文约 4-6 字/秒),过快>7 或过慢<3 都不自然。",
"停顿节奏": "结合语速与文本长度判断停顿/节奏是否合理,过快通常意味吞字、节奏差。",
"整体": "综合以上给出的总体印象分。",
}
# 说明:默认(回译)评审看不到音频,无法覆盖书中「情感表达 / 音色一致性」;这两维需
# 多模态直接听音频,用 --gemini 复现(音色一致性还需参考语音,本 demo 未提供)。
_JUDGE_SYSTEM = """你是严格的 TTS(文本转语音)质量评审专家。
你将拿到:原始参考文本、该文本的期望情感、由 Whisper 对合成语音回译得到的转写文本,
以及从音频客观测得的时长、语速(字/秒)和字错误率(CER)。
请据此对合成语音质量按 Rubric 逐维度打分(1-5 的整数,5 最好):
- 清晰度:转写与原文是否高度一致(漏字/错字/多字越多分越低;CER 越高分越低)。
- 自然度:语速是否接近自然朗读(中文自然朗读约 4-6 字/秒;过快>7 或过慢<3 都不自然)。
- 停顿节奏:结合语速与文本长度,判断停顿/节奏是否合理(过快通常意味着吞字、节奏差)。
- 整体:综合以上给出的总体印象分。
注意:你看不到音频本身,只能基于以上可测特征做保守、可解释的判断。
只输出 JSON,格式:
{"清晰度": {"score": int, "reason": str},
"自然度": {"score": int, "reason": str},
"停顿节奏": {"score": int, "reason": str},
"整体": {"score": int, "reason": str}}
reason 用一句简短中文说明。"""
@dataclass
class RubricResult:
scores: dict # 维度 -> int
reasons: dict # 维度 -> str
raw: str = ""
def judge_rubric(reference: str, emotion: str, hypothesis: str,
duration: float, cer: float, model: Optional[str] = None) -> RubricResult:
"""用评审模型(默认 gpt-5.6-luna)按 Rubric 打分。返回结构化分数 + 点评。
评审 chat 调用支持 OpenRouter 回退(见 get_judge_client_and_model)。"""
chars = len(normalize(reference))
speed = chars / duration if duration > 0 else 0.0
user = (
f"原始参考文本:{reference}\n"
f"期望情感:{emotion}\n"
f"Whisper 回译文本:{hypothesis}\n"
f"音频时长:{duration:.2f} 秒\n"
f"语速:{speed:.2f} 字/秒(参考文本 {chars} 个有效字符)\n"
f"字错误率 CER:{cer:.3f}\n"
)
judge_client, judge_model = get_judge_client_and_model(model or config.JUDGE_MODEL)
resp = judge_client.chat.completions.create(
model=judge_model,
messages=[{"role": "system", "content": _JUDGE_SYSTEM},
{"role": "user", "content": user}],
temperature=0.0,
response_format={"type": "json_object"},
)
raw = resp.choices[0].message.content or "{}"
data = json.loads(raw)
scores, reasons = {}, {}
for dim in RUBRIC_DIMENSIONS:
item = data.get(dim, {})
if isinstance(item, dict):
scores[dim] = int(item.get("score") or 0) # score 缺失或为 null 时按 0 分
reasons[dim] = str(item.get("reason", "")).strip()
else: # 兼容模型直接返回数字(null 按 0 分)
scores[dim] = int(item or 0)
reasons[dim] = ""
return RubricResult(scores=scores, reasons=reasons, raw=raw)
# ---------------------------------------------------------------------------
# 6) 可选:Gemini 多模态音频评审(书中方案)。用 REST,避免额外 SDK 依赖。
# ---------------------------------------------------------------------------
def _resolve_gemini_model(api_key: str) -> str:
"""探测当前可用的 Gemini 模型,避免默认名过期。"""
import urllib.request
url = f"https://generativelanguage.googleapis.com/v1beta/models?key={api_key}"
try:
with urllib.request.urlopen(url, timeout=20) as r:
data = json.loads(r.read())
names = [m["name"].split("/")[-1] for m in data.get("models", [])
if "generateContent" in m.get("supportedGenerationMethods", [])]
# 优先默认的 gemini-3.5-flash(已验证支持音频输入),再退到 pro / 旧 flash 系列。
for want in (config.GEMINI_MODEL_DEFAULT, "gemini-3.5-flash",
"gemini-2.5-pro", "gemini-2.5-flash", "gemini-flash-latest"):
if want in names:
return want
# 退而求其次:任意非 tts/image 的可用模型
for n in names:
if "tts" not in n and "image" not in n and "embedding" not in n:
return n
except Exception:
pass
return config.GEMINI_MODEL_DEFAULT
def judge_gemini_audio(reference: str, emotion: str, audio_path: str) -> RubricResult:
"""把合成音频 + 原文 + Rubric 一起交给 Gemini 多模态直接「听」并打分。
需要 GEMINI_API_KEY。默认关闭;--gemini 开启。失败抛异常由上层记为失败。
"""
import urllib.request
key = os.environ.get("GEMINI_API_KEY", "").strip()
if not key:
raise RuntimeError("缺少 GEMINI_API_KEY,无法使用 Gemini 音频评审。")
model = _resolve_gemini_model(key)
with open(audio_path, "rb") as f:
audio_b64 = base64.b64encode(f.read()).decode()
prompt = (
"你是 TTS 质量评审专家。请直接聆听下面这段合成语音,对照原始文本与期望情感,"
"按 1-5 分为四个维度打分并给出简短理由,只输出 JSON:"
'{"清晰度":{"score":int,"reason":str},"自然度":{"score":int,"reason":str},'
'"停顿节奏":{"score":int,"reason":str},"整体":{"score":int,"reason":str}}\n'
f"原始文本:{reference}\n期望情感:{emotion}"
)
body = {
"contents": [{"parts": [
{"text": prompt},
{"inline_data": {"mime_type": "audio/mp3", "data": audio_b64}},
]}],
"generationConfig": {"temperature": 0.0, "responseMimeType": "application/json"},
}
url = (f"https://generativelanguage.googleapis.com/v1beta/models/"
f"{model}:generateContent?key={key}")
req = urllib.request.Request(
url, data=json.dumps(body).encode(),
headers={"Content-Type": "application/json"}, method="POST",
)
with urllib.request.urlopen(req, timeout=90) as r:
data = json.loads(r.read())
# Gemini 在安全拦截时不返回 candidates(或 candidate 无 content/parts),
# 防御式取值并给出带 promptFeedback 的清晰错误,交由上层记为该条失败。
candidates = data.get("candidates") or []
parts = []
if candidates:
parts = (candidates[0].get("content") or {}).get("parts") or []
if not parts or not parts[0].get("text"):
raise RuntimeError(f"Gemini 未返回评审文本:{data.get('promptFeedback') or data}")
text = parts[0]["text"]
parsed = json.loads(text)
scores, reasons = {}, {}
# 评审 JSON 的 score 字段缺失或为 null 时按 0 分处理,与 judge_rubric 一致
for dim in RUBRIC_DIMENSIONS:
item = parsed.get(dim, {})
scores[dim] = int(item.get("score") or 0) if isinstance(item, dict) else int(item or 0)
reasons[dim] = str(item.get("reason", "")).strip() if isinstance(item, dict) else ""
return RubricResult(scores=scores, reasons=reasons, raw=text)