test_judge_robustness.py
test_judge_robustness.py
"""
Regression tests for judge-response robustness (实验 6-5 TTS 质量评估).
Covers two failure classes on LLM/Gemini judge responses:
- judge_rubric: judge returns "score": null (or a bare null dimension) -> int(None) TypeError
- judge_gemini_audio: safety-blocked Gemini responses have no
candidates/content/parts -> KeyError/IndexError instead of a clear error
Network is stubbed: the OpenAI-compatible judge client is replaced with a fake,
and urllib.request.urlopen is monkeypatched for the Gemini REST call.
"""
import io
import json
import pytest
import pipeline
class _FakeMessage:
content = "{}"
class _FakeChoice:
message = _FakeMessage()
class _FakeResp:
choices = [_FakeChoice()]
class _FakeCompletions:
@staticmethod
def create(**kwargs):
return _FakeResp()
class _FakeChat:
completions = _FakeCompletions()
class _FakeClient:
chat = _FakeChat()
def _stub_judge(monkeypatch, payload: dict):
_FakeMessage.content = json.dumps(payload, ensure_ascii=False)
monkeypatch.setattr(
pipeline, "get_judge_client_and_model", lambda model=None: (_FakeClient(), "fake-judge"))
def test_judge_rubric_tolerates_null_score(monkeypatch):
"""'score': null in a dimension dict is scored 0, not int(None) TypeError."""
_stub_judge(monkeypatch, {
"清晰度": {"score": None, "reason": "无法判断"},
"自然度": {"score": 4, "reason": "语速正常"},
"停顿节奏": {"score": 3},
"整体": {"score": 5, "reason": "总体可用"},
})
rub = pipeline.judge_rubric("原文文本", "中性", "回译文本", 3.0, 0.05)
assert rub.scores["清晰度"] == 0
assert rub.scores["自然度"] == 4
assert rub.scores["整体"] == 5
def test_judge_rubric_tolerates_null_dimension(monkeypatch):
"""A bare null dimension (non-dict) is scored 0, not int(None) TypeError."""
_stub_judge(monkeypatch, {"清晰度": None, "自然度": 4, "停顿节奏": 3, "整体": 5})
rub = pipeline.judge_rubric("原文文本", "中性", "回译文本", 3.0, 0.05)
assert rub.scores["清晰度"] == 0
assert rub.scores["自然度"] == 4
class _FakeHTTPResp(io.BytesIO):
def __enter__(self):
return self
def __exit__(self, *args):
return False
def _stub_gemini(monkeypatch, payload: dict):
monkeypatch.setenv("GEMINI_API_KEY", "fake-key-for-test")
monkeypatch.setattr(pipeline, "_resolve_gemini_model", lambda key: "gemini-fake")
monkeypatch.setattr("urllib.request.urlopen",
lambda req, timeout=None: _FakeHTTPResp(json.dumps(payload).encode()))
@pytest.mark.parametrize("payload", [
{"promptFeedback": {"blockReason": "SAFETY"}}, # prompt 被拦截:无 candidates
{"candidates": []}, # 生成被拦截:空 candidates
{"candidates": [{"finishReason": "SAFETY", "index": 0}]}, # candidate 无 content
])
def test_judge_gemini_audio_blocked_raises_clear_error(monkeypatch, tmp_path, payload):
"""Blocked/empty Gemini responses raise a clear RuntimeError, not KeyError/IndexError."""
_stub_gemini(monkeypatch, payload)
audio = tmp_path / "a.mp3"
audio.write_bytes(b"\xff\xfb" + b"\x00" * 256)
with pytest.raises(RuntimeError, match="Gemini 未返回评审文本"):
pipeline.judge_gemini_audio("原文", "中性", str(audio))
def test_judge_gemini_audio_parses_valid_response(monkeypatch, tmp_path):
"""A normal Gemini response still parses (defensive navigation keeps working)."""
inner = json.dumps({"清晰度": {"score": 4, "reason": "ok"}, "自然度": 4,
"停顿节奏": None, "整体": {"score": 5}}, ensure_ascii=False)
_stub_gemini(monkeypatch, {
"candidates": [{"content": {"parts": [{"text": inner}]}}],
})
audio = tmp_path / "a.mp3"
audio.write_bytes(b"\xff\xfb" + b"\x00" * 256)
rub = pipeline.judge_gemini_audio("原文", "中性", str(audio))
assert rub.scores["清晰度"] == 4
assert rub.scores["停顿节奏"] == 0 # null score -> 0
assert rub.scores["整体"] == 5
if __name__ == "__main__":
pytest.main([__file__, "-v"])