fix(analysis): 更稳健地从 LLM 输出提取 JSON，并对截断输出用 json-repair 兜底

goodprice/analysis/json_extract.py

@@ -0,0 +1,103 @@
+"""从 LLM 文本输出中稳健地提取 JSON 对象。
+
+LLM 常把 JSON 包在 ```json 围栏里、或在前后夹带解释文字，也偶尔多出尾随逗号。
+这里先用正则剥掉围栏，再用括号配平扫描取出第一个完整对象（能正确跳过字符串内
+的括号与转义）；比原先的 ``find("{")``/``rfind("}")`` 更稳：后者一旦输出里出现
+多个对象、或字符串中含有大括号，就会取错范围。
+
+标准解析仍失败时，若安装了可选依赖 ``json-repair``，则交给它修复畸形或截断的
+JSON（例如输出被 max_tokens 截断）。
+"""
+from __future__ import annotations
+
+import json
+import logging
+import re
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+_FENCE_RE = re.compile(r"```[a-zA-Z0-9_-]*\s*(.*?)\s*```", re.DOTALL)
+
+
+def _strip_fence(text: str) -> str:
+    match = _FENCE_RE.search(text)
+    if match:
+        return match.group(1)
+    # 只有开头围栏、没有结尾（输出被截断）时，去掉开头标记
+    if text.lstrip().startswith("```"):
+        return re.sub(r"^\s*```[a-zA-Z0-9_-]*\s*", "", text, count=1)
+    return text
+
+
+def _find_balanced(text: str) -> str | None:
+    """返回第一个括号配平的 ``{...}`` 片段；跳过字符串内的括号与转义。"""
+    start = text.find("{")
+    if start == -1:
+        return None
+    depth = 0
+    in_string = False
+    escaped = False
+    for i in range(start, len(text)):
+        ch = text[i]
+        if in_string:
+            if escaped:
+                escaped = False
+            elif ch == "\\":
+                escaped = True
+            elif ch == '"':
+                in_string = False
+            continue
+        if ch == '"':
+            in_string = True
+        elif ch == "{":
+            depth += 1
+        elif ch == "}":
+            depth -= 1
+            if depth == 0:
+                return text[start : i + 1]
+    return None
+
+
+def _remove_trailing_commas(text: str) -> str:
+    return re.sub(r",(\s*[}\]])", r"\1", text)
+
+
+def _try_repair(text: str) -> dict[str, Any] | None:
+    """标准解析失败时，尝试用可选依赖 json-repair 修复畸形/截断的 JSON。"""
+    try:
+        from json_repair import repair_json
+    except ImportError:
+        return None
+    try:
+        repaired = repair_json(text, return_objects=True)
+    except Exception as exc:  # noqa: BLE001
+        logger.debug("json-repair 修复失败: %s", exc)
+        return None
+    if isinstance(repaired, dict) and repaired:
+        return repaired
+    return None
+
+
+def extract_json_object(raw: str) -> dict[str, Any]:
+    """从任意 LLM 文本中提取第一个 JSON 对象；失败时抛 ``ValueError``。"""
+    text = (raw or "").strip()
+    if not text:
+        raise ValueError(f"LLM 输出为空: {raw!r}")
+
+    candidate = _strip_fence(text)
+    block = _find_balanced(candidate)
+    if block is not None:
+        for attempt in (block, _remove_trailing_commas(block)):
+            try:
+                data = json.loads(attempt)
+            except json.JSONDecodeError:
+                continue
+            if isinstance(data, dict):
+                return data
+
+    repaired = _try_repair(candidate)
+    if repaired is not None:
+        return repaired
+
+    raise ValueError(f"LLM 输出中没有可解析的 JSON: {raw!r}")

goodprice/analysis/judge.py

@@ -1,12 +1,11 @@
 # Jev 判断层（实验性）：需求匹配与批量性价比的类型化逐件评估。
 # 与默认路径的区别：每次判断独立成请求并附带置信度，单件失败不拖累整批；
 # 置信度低于阈值的判断由调用方回落到原 LLMClient 路径（宁多勿漏）。
-import json
 import logging
-import re
 from dataclasses import dataclass
 from typing import Any, Callable, Optional
 
+from goodprice.analysis.json_extract import extract_json_object
 from goodprice.analysis.llm import LLMClient
 
 logger = logging.getLogger(__name__)
@@ -40,19 +39,8 @@
 )
 
 
-def _extract_json(raw: str) -> dict[str, Any]:
-    text = (raw or "").strip()
-    if text.startswith("```"):
-        text = re.sub(r"^```[a-zA-Z]*\s*", "", text)
-        text = re.sub(r"\s*```$", "", text)
-    start, end = text.find("{"), text.rfind("}")
-    if start == -1 or end == -1:
-        raise ValueError(f"LLM 输出中没有 JSON: {raw!r}")
-    return json.loads(text[start : end + 1])
-
-
 def parse_requirement_typed(raw: str) -> dict[str, Any]:
-    data = _extract_json(raw)
+    data = extract_json_object(raw)
     matched = data.get("matched")
     if not isinstance(matched, bool):
         raise ValueError(f"类型化需求判断缺少布尔 matched: {raw!r}")
@@ -64,7 +52,7 @@ def parse_requirement_typed(raw: str) -> dict[str, Any]:
 
 
 def parse_value_typed(raw: str) -> dict[str, Any]:
-    data = _extract_json(raw)
+    data = extract_json_object(raw)
     try:
         score = max(1, min(10, int(data.get("value_score", 0))))
     except (TypeError, ValueError):

goodprice/analysis/llm.py

@@ -1,11 +1,11 @@
 import json
 import logging
-import re
 import time
 from typing import Any, Optional
 
 import httpx
 
+from goodprice.analysis.json_extract import extract_json_object
 from goodprice.analysis.prompts import (
     BATCH_VALUE_SYSTEM_PROMPT,
     BATCH_VALUE_USER_TEMPLATE,
@@ -29,19 +29,8 @@
 )
 
 
-def _extract_json(raw: str) -> dict[str, Any]:
-    text = (raw or "").strip()
-    if text.startswith("```"):
-        text = re.sub(r"^```[a-zA-Z]*\s*", "", text)
-        text = re.sub(r"\s*```$", "", text)
-    start, end = text.find("{"), text.rfind("}")
-    if start == -1 or end == -1:
-        raise ValueError(f"LLM 输出中没有 JSON: {raw!r}")
-    return json.loads(text[start : end + 1])
-
-
 def parse_analysis_json(raw: str) -> dict[str, Any]:
-    data = _extract_json(raw)
+    data = extract_json_object(raw)
     score = max(1, min(10, int(data.get("condition_score", 0))))
     defects = [str(d) for d in data.get("defects", [])][:10]
     return {
@@ -53,7 +42,7 @@ def parse_analysis_json(raw: str) -> dict[str, Any]:
 
 
 def parse_requirement_json(raw: str) -> dict[str, Any]:
-    data = _extract_json(raw)
+    data = extract_json_object(raw)
     matched = data.get("matched")
     if not isinstance(matched, bool):
         raise ValueError(f"需求判断输出缺少布尔 matched: {raw!r}")
@@ -62,7 +51,7 @@ def parse_requirement_json(raw: str) -> dict[str, Any]:
 
 def parse_batch_value_json(raw: str) -> dict[str, Any]:
     """解析批量性价比输出：{"items": [{"id", "value_score", "reason"}], "best": id}。"""
-    data = _extract_json(raw)
+    data = extract_json_object(raw)
     items = data.get("items")
     if not isinstance(items, list) or not items:
         raise ValueError(f"批量性价比输出缺少 items: {raw!r}")

pyproject.toml

@@ -22,6 +22,7 @@ dependencies = [
   "httpx>=0.27",
   "jinja2>=3.1",
   "python-multipart>=0.0.9",
+  "json-repair>=0.30",
 ]
 
 [project.optional-dependencies]

requirements.lock

@@ -173,6 +173,10 @@ jinja2==3.1.6 \
     --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \
     --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67
     # via good-price
+json-repair==0.63.5 \
+    --hash=sha256:0e39b3066f7829b4bd7fc614e463cfd4ff7a4c46f82f5b65431fc11e29a74c81 \
+    --hash=sha256:75a6a1a4ea954935dd3fb69b6e49ac0e3e3d69b4290127a0c8d37af93b709f3a
+    # via good-price
 markupsafe==3.0.3 \
     --hash=sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a \
     --hash=sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf \

tests/test_json_extract.py

@@ -0,0 +1,75 @@
+import pytest
+
+from goodprice.analysis.json_extract import extract_json_object
+from goodprice.analysis.judge import parse_requirement_typed
+from goodprice.analysis.llm import parse_analysis_json, parse_requirement_json
+
+
+def test_plain_object():
+    assert extract_json_object('{"a": 1}') == {"a": 1}
+
+
+def test_fenced_with_language_tag():
+    raw = '```json\n{"condition_score": 3}\n```'
+    assert extract_json_object(raw) == {"condition_score": 3}
+
+
+def test_surrounding_prose():
+    raw = '好的，结论如下：{"matched": true, "reason": "ok"} 希望有帮助'
+    assert extract_json_object(raw) == {"matched": True, "reason": "ok"}
+
+
+def test_braces_inside_string_do_not_break_scan():
+    raw = '前言 { "note": "价格 {面议}", "n": 1 } 结尾 }'
+    assert extract_json_object(raw) == {"note": "价格 {面议}", "n": 1}
+
+
+def test_picks_first_complete_object_when_multiple():
+    raw = '{"a": 1} 以及 {"b": 2}'
+    assert extract_json_object(raw) == {"a": 1}
+
+
+def test_tolerates_trailing_comma():
+    assert extract_json_object('{"a": 1, "b": [1, 2,],}') == {"a": 1, "b": [1, 2]}
+
+
+def test_truncated_fence_still_parses():
+    raw = '```json\n{"value_score": 8}'
+    assert extract_json_object(raw) == {"value_score": 8}
+
+
+def test_no_json_raises():
+    with pytest.raises(ValueError):
+        extract_json_object("抱歉，我无法判断")
+
+
+def test_repair_fallback_for_truncated_json():
+    pytest.importorskip("json_repair")
+    raw = '```json\n{"matched": true, "reason": "x"'
+    assert extract_json_object(raw) == {"matched": True, "reason": "x"}
+
+
+def test_repair_never_invents_object_from_prose():
+    # 即使装了 json-repair，纯文本也不应被强行解析成对象
+    with pytest.raises(ValueError):
+        extract_json_object("抱歉，我无法判断")
+
+
+def test_empty_raises():
+    with pytest.raises(ValueError):
+        extract_json_object("   ")
+
+
+def test_parsers_use_shared_extractor():
+    verdict = parse_analysis_json(
+        '```json\n{"condition_score": 99, "defects": [], "recommended": false, "reason": "x"}\n```'
+    )
+    assert verdict["condition_score"] == 10
+    assert parse_requirement_json('说明：{"matched": true, "reason": "ok"}') == {
+        "matched": True,
+        "reason": "ok",
+    }
+    typed = parse_requirement_typed(
+        '```json\n{"matched": true, "probability": 0.9, "reason": "ok"}\n```'
+    )
+    assert typed["matched"] is True