demo.py
demo.py
"""实验 6-5:全自动 TTS 质量评估流水线 —— 一条命令跑通。
python demo.py # 默认 4 个 OpenAI 配置 x 4 条语料
python demo.py --providers openai,minimax # 跨服务商横向对比
python demo.py --text '一段话' # 自定义文本
python demo.py --gemini # 评审改用 Gemini 多模态直接听音频(需 GEMINI_API_KEY)
python demo.py --quick # 只用前 2 条语料,快速冒烟
python demo.py --list-providers # 离线:查看 provider 及配置状态
python demo.py --dump-rubric # 离线:查看 Rubric 维度定义
流程:多 provider TTS 合成 -> ffprobe 时长 -> Whisper 回译 -> CER/字准确率
-> LLM/Gemini Rubric 打分 -> 打印逐条明细 + 配置对比汇总表。
幂等:音频写入 output/ 并复用(除非 --fresh)。完整参数见 `python demo.py --help`。
"""
import argparse
import json
import os
import sys
import time
import traceback
from statistics import mean
import config
import pipeline
OUT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "output")
def load_env():
"""极简 .env 加载(不引第三方依赖)。"""
path = os.path.join(os.path.dirname(os.path.abspath(__file__)), ".env")
if os.path.exists(path):
for line in open(path, encoding="utf-8"):
line = line.strip()
if line and not line.startswith("#") and "=" in line:
k, v = line.split("=", 1)
os.environ.setdefault(k.strip(), v.strip())
def audio_path(cfg_name: str, sample_id: str) -> str:
return os.path.join(OUT_DIR, f"{cfg_name}__{sample_id}.mp3")
def evaluate_one(cfg, sample, use_gemini: bool, fresh: bool,
judge_model: str = None) -> dict:
"""对单个 (配置, 语料) 跑完整链路。任一步失败返回 error 记录,不抛出。"""
rec = {"config": cfg.name, "sample": sample.id, "challenge": sample.challenge,
"provider": getattr(cfg, "provider", "openai"), "ok": False, "error": None}
path = audio_path(cfg.name, sample.id)
try:
# 1) 合成(幂等:已存在且非 fresh 则复用)
if fresh or not os.path.exists(path) or os.path.getsize(path) == 0:
pipeline.synthesize(cfg, sample.text, path)
# 2) 时长
dur = pipeline.probe_duration(path)
# 3) 回译
hyp = pipeline.transcribe(path)
# 4) CER / 字准确率
er = pipeline.char_error_rate(sample.text, hyp)
# 5) Rubric 打分
if use_gemini:
rub = pipeline.judge_gemini_audio(sample.text, sample.emotion, path)
else:
rub = pipeline.judge_rubric(sample.text, sample.emotion, hyp, dur, er.cer,
model=judge_model)
rec.update({
"ok": True, "duration": dur, "hypothesis": hyp,
"cer": er.cer, "accuracy": er.accuracy,
"speed": (er.ref_len / dur) if dur else 0.0,
"scores": rub.scores, "reasons": rub.reasons,
})
except Exception as e: # 单条失败不影响整表
rec["error"] = f"{type(e).__name__}: {e}"
return rec
def fmt(x, nd=2):
return f"{x:.{nd}f}" if isinstance(x, (int, float)) else str(x)
def print_detail(rec, sample_text):
head = f"[{rec['config']} | {rec['sample']}] {rec['challenge']}"
if not rec["ok"]:
print(f" {head}\n !! 失败: {rec['error']}")
return
print(f" {head}")
print(f" 原文 : {sample_text}")
print(f" 回译 : {rec['hypothesis']}")
print(f" 时长 : {fmt(rec['duration'])}s 语速: {fmt(rec['speed'])} 字/秒"
f" CER: {fmt(rec['cer'],3)} 字准确率: {fmt(rec['accuracy']*100,1)}%")
s, r = rec["scores"], rec["reasons"]
for dim in pipeline.RUBRIC_DIMENSIONS:
print(f" {dim:<4}: {s.get(dim,'-')}/5 {r.get(dim,'')}")
def summarize(records):
"""按配置聚合:平均 CER、平均字准确率、各 Rubric 维度均分、成功数。"""
by_cfg = {}
for rec in records:
by_cfg.setdefault(rec["config"], []).append(rec)
rows = []
for cfg_name, recs in by_cfg.items():
ok = [r for r in recs if r["ok"]]
row = {"config": cfg_name, "n_ok": len(ok), "n": len(recs)}
if ok:
row["cer"] = mean(r["cer"] for r in ok)
row["accuracy"] = mean(r["accuracy"] for r in ok)
for dim in pipeline.RUBRIC_DIMENSIONS:
row[dim] = mean(r["scores"].get(dim, 0) for r in ok)
rows.append(row)
# 按整体分降序、CER 升序排序
rows.sort(key=lambda x: (-x.get("整体", 0), x.get("cer", 1)))
return rows
def print_table(rows):
cols = ["整体", "清晰度", "自然度", "停顿节奏"]
header = (f"{'配置':<18}{'成功':>6}{'字准确率':>10}{'CER':>8}"
+ "".join(f"{c:>9}" for c in cols))
print(header)
print("-" * 74)
for r in rows:
ok_str = f"{r['n_ok']}/{r['n']}"
if not r.get("n_ok"):
print(f"{r['config']:<18}{ok_str:>6} (全部失败)")
continue
acc = f"{r['accuracy']*100:.1f}%"
line = f"{r['config']:<18}{ok_str:>6}{acc:>10}{r['cer']:>8.3f}"
line += "".join(f"{r.get(c,0):>9.2f}" for c in cols)
print(line)
def print_providers():
"""离线打印所有可用 TTS provider 及其配置状态(无需任何 API key)。"""
print("可用 TTS provider(书中:OpenAI / ElevenLabs / Fish Audio / Minimax / 豆包):\n")
for key, p in config.PROVIDERS.items():
state = "已配置" if p.configured() else "未配置"
env = " + ".join(p.env)
print(f" [{key}] {p.label} ({state};需 {env})")
print(f" {p.note}")
print("\n用 --providers openai,minimax 选择跨服务商横向对比(默认仅 OpenAI)。")
print("非 OpenAI provider 需各自的 key(见 env.example);缺 key 时该行记为失败,不中断整表。")
def print_rubric():
"""离线打印 Rubric 维度定义(无需任何 API key)。"""
print("TTS 质量评估 Rubric(1-5 分,5 最好):\n")
for dim in pipeline.RUBRIC_DIMENSIONS:
print(f" {dim}:{pipeline.RUBRIC_DESCRIPTIONS.get(dim, '')}")
print("\n默认(Whisper 回译 + LLM)评审基于「转写文本 + 时长 + 语速 + CER」保守打分;")
print("--gemini 让多模态模型直接听音频,可覆盖书中「情感表达 / 音色一致性」维度。")
def main():
global OUT_DIR
ap = argparse.ArgumentParser(
description="全自动 TTS 质量评估流水线(实验 6-5):多 provider 合成 + 多模态 LLM Rubric 评审",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="示例:\n"
" python demo.py 默认 4 个 OpenAI 配置 × 4 条语料\n"
" python demo.py --providers openai,minimax 跨服务商横向对比\n"
" python demo.py --text '今天天气不错' --gemini 自定义文本 + Gemini 多模态评审\n"
" python demo.py --list-providers 离线查看 provider 及配置状态\n"
" python demo.py --dump-rubric 离线查看 Rubric 维度定义",
)
ap.add_argument("--text", metavar="文本",
help="用一段自定义文本替换测试语料库(只评这一句)")
ap.add_argument("--providers", metavar="列表",
help="逗号分隔的 provider(openai,elevenlabs,fishaudio,minimax,doubao),"
"每个取代表性配置做横向对比;默认仅 OpenAI 的多配置")
ap.add_argument("--judge-model", metavar="模型", dest="judge_model",
help=f"覆盖 LLM 评审模型(默认 {config.JUDGE_MODEL});--gemini 时不生效")
ap.add_argument("--output", metavar="目录",
help=f"输出目录(音频 + results.json),默认 {OUT_DIR}")
ap.add_argument("--extra", action="store_true", help="额外加入 gpt-4o-mini-tts 配置")
ap.add_argument("--gemini", action="store_true", help="用 Gemini 多模态直接听音频评审(需 GEMINI_API_KEY)")
ap.add_argument("--quick", action="store_true", help="只用前 2 条语料快速冒烟")
ap.add_argument("--fresh", action="store_true", help="忽略已有音频,全部重新合成")
ap.add_argument("--list-providers", action="store_true", dest="list_providers",
help="离线打印所有 TTS provider 及配置状态后退出(无需 key)")
ap.add_argument("--dump-rubric", action="store_true", dest="dump_rubric",
help="离线打印 Rubric 维度定义后退出(无需 key)")
args = ap.parse_args()
load_env()
# 离线路径:不联网、不需要任何 key,打印后直接退出。
if args.list_providers:
print_providers()
return
if args.dump_rubric:
print_rubric()
return
if args.output:
OUT_DIR = os.path.abspath(args.output)
os.makedirs(OUT_DIR, exist_ok=True)
if not os.environ.get("OPENAI_API_KEY", "").strip():
print("错误:缺少 OPENAI_API_KEY(回译/默认评审需要)。请 export 或写入 .env 后重试。",
file=sys.stderr)
sys.exit(1)
# 选择待对比的配置:--providers 优先(跨服务商),否则默认 OpenAI 多配置。
if args.providers:
configs = []
for key in [p.strip() for p in args.providers.split(",") if p.strip()]:
if key not in config.PROVIDER_CONFIGS:
print(f"错误:未知 provider {key!r}。可用:{', '.join(config.PROVIDER_CONFIGS)}",
file=sys.stderr)
sys.exit(1)
configs.append(config.PROVIDER_CONFIGS[key])
else:
configs = list(config.TTS_CONFIGS)
if args.extra:
configs += config.EXTRA_CONFIGS
if args.text:
corpus = [config.Sample(id="custom", text=args.text,
challenge="自定义文本", emotion="中性")]
else:
corpus = config.CORPUS[:2] if args.quick else config.CORPUS
judge_model = args.judge_model or config.JUDGE_MODEL
mode = ("Gemini 多模态音频评审" if args.gemini
else f"Whisper 回译 + LLM Rubric({judge_model})")
providers_used = sorted({getattr(c, "provider", "openai") for c in configs})
print("=" * 72)
print(f"实验 6-5:全自动 TTS 质量评估流水线")
print(f"评审模式:{mode}")
print(f"参与 provider:{', '.join(providers_used)}")
print(f"配置数:{len(configs)} 语料数:{len(corpus)} "
f"共 {len(configs)*len(corpus)} 条待评估")
print("=" * 72)
records = []
t0 = time.time()
for cfg in configs:
print(f"\n### 配置 {cfg.name} (provider={getattr(cfg,'provider','openai')}, "
f"model={cfg.model}, voice={cfg.voice}, speed={cfg.speed})")
for sample in corpus:
rec = evaluate_one(cfg, sample, args.gemini, args.fresh,
judge_model=None if args.gemini else args.judge_model)
print_detail(rec, sample.text)
records.append(rec)
rows = summarize(records)
print("\n" + "=" * 72)
print("配置对比汇总(按 整体 分降序)")
print("=" * 72)
print_table(rows)
ok = sum(1 for r in records if r["ok"])
print(f"\n完成:{ok}/{len(records)} 条成功,耗时 {time.time()-t0:.1f}s。")
# 落盘结构化结果,便于二次分析
out_json = os.path.join(OUT_DIR, "results.json")
with open(out_json, "w", encoding="utf-8") as f:
json.dump({"records": records, "summary": rows}, f,
ensure_ascii=False, indent=2)
print(f"明细结果已写入 {out_json}")
if __name__ == "__main__":
main()