Просмотр исходного кода

fix: align review pipeline metrics with save-review contract

lingfengQAQ 2 месяцев назад
Родитель
Сommit
9304aa425f

+ 25 - 3
webnovel-writer/references/review-schema.md

@@ -28,6 +28,28 @@
 
 ## 指标沉淀
 
-每次审查写入 `index.db.review_metrics`:
-- `chapter, issues_count, blocking_count, categories, timestamp`
-- 用于趋势观测,不用于 gate 决策
+统一审查 agent 的原始输出保存为 `review_results.json`,保留完整 `issues` 列表。
+
+随后由 `review-pipeline` 生成 `review_metrics.json`,用于写入 `index.db.review_metrics`。
+该文件同时包含两类信息:
+
+- **落库兼容字段**:
+  - `start_chapter`
+  - `end_chapter`
+  - `overall_score`(由问题严重度推导的兼容分)
+  - `dimension_scores`
+  - `severity_counts`
+  - `critical_issues`
+  - `report_file`
+  - `notes`
+- **v6 观测字段**:
+  - `chapter`
+  - `issues_count`
+  - `blocking_count`
+  - `categories`
+  - `timestamp`
+
+说明:
+- `review_metrics` 表仍沿用现有 dashboard / trend / context 消费的兼容 schema。
+- `overall_score` 仅用于趋势观测与排序,不替代原始 issue 清单。
+- gate 决策仍以 `blocking=true` 和 issue 明细为准。

+ 78 - 2
webnovel-writer/scripts/data_modules/review_schema.py

@@ -16,6 +16,30 @@ VALID_CATEGORIES = {
     "continuity", "setting", "character", "timeline",
     "ai_flavor", "logic", "pacing", "other",
 }
+SCORE_CATEGORIES = (
+    "continuity",
+    "setting",
+    "character",
+    "timeline",
+    "ai_flavor",
+    "logic",
+    "pacing",
+    "other",
+)
+SEVERITY_PENALTIES = {
+    "critical": 35.0,
+    "high": 15.0,
+    "medium": 6.0,
+    "low": 2.0,
+}
+
+
+def _clamp_score(value: float) -> float:
+    return round(max(0.0, min(100.0, value)), 2)
+
+
+def _issue_penalty(issue: "ReviewIssue") -> float:
+    return float(SEVERITY_PENALTIES.get(issue.severity, SEVERITY_PENALTIES["medium"]))
 
 
 @dataclass
@@ -58,6 +82,49 @@ class ReviewResult:
     def has_blocking(self) -> bool:
         return self.blocking_count > 0
 
+    @property
+    def severity_counts(self) -> Dict[str, int]:
+        counts = {level: 0 for level in ("critical", "high", "medium", "low")}
+        for issue in self.issues:
+            severity = issue.severity if issue.severity in counts else "medium"
+            counts[severity] += 1
+        return counts
+
+    @property
+    def categories(self) -> List[str]:
+        return sorted(set(i.category for i in self.issues))
+
+    @property
+    def critical_issues(self) -> List[str]:
+        return [
+            issue.description
+            for issue in self.issues
+            if issue.severity == "critical" and issue.description
+        ]
+
+    def _build_dimension_scores(self) -> Dict[str, float]:
+        scores = {category: 100.0 for category in SCORE_CATEGORIES}
+        for issue in self.issues:
+            category = issue.category if issue.category in scores else "other"
+            scores[category] = _clamp_score(scores[category] - _issue_penalty(issue))
+        return scores
+
+    def _build_notes(self, categories: List[str]) -> str:
+        parts: List[str] = []
+        if self.summary:
+            parts.append(self.summary)
+        parts.append(f"issues={self.issues_count}")
+        parts.append(f"blocking={self.blocking_count}")
+        if categories:
+            parts.append("categories=" + ",".join(categories))
+        return " | ".join(parts)
+
+    def _calculate_overall_score(self) -> float:
+        score = 100.0
+        for issue in self.issues:
+            score -= _issue_penalty(issue)
+        return _clamp_score(score)
+
     def to_dict(self) -> Dict[str, Any]:
         return {
             "chapter": self.chapter,
@@ -68,10 +135,19 @@ class ReviewResult:
             "summary": self.summary,
         }
 
-    def to_metrics_dict(self) -> Dict[str, Any]:
-        categories = sorted(set(i.category for i in self.issues))
+    def to_metrics_dict(self, report_file: str = "") -> Dict[str, Any]:
+        categories = self.categories
+        severity_counts = self.severity_counts
         return {
             "chapter": self.chapter,
+            "start_chapter": self.chapter,
+            "end_chapter": self.chapter,
+            "overall_score": self._calculate_overall_score(),
+            "dimension_scores": self._build_dimension_scores(),
+            "severity_counts": severity_counts,
+            "critical_issues": self.critical_issues,
+            "report_file": report_file,
+            "notes": self._build_notes(categories),
             "issues_count": self.issues_count,
             "blocking_count": self.blocking_count,
             "categories": categories,

+ 23 - 4
webnovel-writer/scripts/data_modules/tests/test_prompt_integrity.py

@@ -222,10 +222,7 @@ KNOWN_DELETED_FILES = [
     "webnovel-resume",
 ]
 
-# webnovel-review SKILL.md 仍有大量 workflow 命令,需单独重构(Phase 1 遗漏)
-_KNOWN_CLI_EXCEPTIONS = {
-    "webnovel-review": {"workflow"},
-}
+_KNOWN_CLI_EXCEPTIONS = {}
 
 
 @pytest.mark.parametrize("prompt_file", ALL_PROMPT_FILES, ids=lambda f: f.name)
@@ -234,3 +231,25 @@ def test_no_stale_references(prompt_file: Path):
     text = _read_text(prompt_file)
     found = [name for name in KNOWN_DELETED_FILES if name in text]
     assert not found, f"{prompt_file.name}: 残留引用已删除文件 {found}"
+
+
+def test_webnovel_review_skill_uses_unified_reviewer_pipeline():
+    """webnovel-review 必须与 webnovel-write 使用同一套 reviewer + review-pipeline 链路。"""
+    skill_text = _read_text(SKILLS_DIR / "webnovel-review" / "SKILL.md")
+
+    assert "`reviewer`" in skill_text
+    assert "review-pipeline" in skill_text
+    assert ".webnovel/tmp/review_results.json" in skill_text
+    assert ".webnovel/tmp/review_metrics.json" in skill_text
+
+    for legacy_agent in (
+        "consistency-checker",
+        "continuity-checker",
+        "ooc-checker",
+        "reader-pull-checker",
+        "high-point-checker",
+        "pacing-checker",
+    ):
+        assert legacy_agent not in skill_text
+
+    assert " workflow " not in skill_text

+ 9 - 0
webnovel-writer/scripts/data_modules/tests/test_review_schema.py

@@ -102,7 +102,16 @@ def test_review_result_to_metrics_dict():
     )
     metrics = result.to_metrics_dict()
     assert metrics["chapter"] == 10
+    assert metrics["start_chapter"] == 10
+    assert metrics["end_chapter"] == 10
     assert metrics["issues_count"] == 2
     assert metrics["blocking_count"] == 1
     assert "continuity" in metrics["categories"]
     assert "ai_flavor" in metrics["categories"]
+    assert metrics["severity_counts"]["critical"] == 1
+    assert metrics["severity_counts"]["high"] == 1
+    assert metrics["critical_issues"] == ["d1"]
+    assert metrics["report_file"] == ""
+    assert metrics["overall_score"] < 100
+    assert metrics["dimension_scores"]["continuity"] < 100
+    assert metrics["dimension_scores"]["ai_flavor"] < 100

+ 7 - 1
webnovel-writer/scripts/data_modules/tests/test_webnovel_unified_cli.py

@@ -268,8 +268,14 @@ def test_review_pipeline_builds_artifacts(tmp_path):
     assert payload["review_result"]["blocking_count"] == 1
     assert payload["review_result"]["has_blocking"] is True
     assert payload["review_result"]["issues_count"] == 2
+    assert payload["metrics"]["start_chapter"] == 20
+    assert payload["metrics"]["end_chapter"] == 20
     assert payload["metrics"]["issues_count"] == 2
     assert payload["metrics"]["blocking_count"] == 1
+    assert payload["metrics"]["severity_counts"]["critical"] == 1
+    assert payload["metrics"]["severity_counts"]["medium"] == 1
+    assert payload["metrics"]["critical_issues"] == ["时间线回跳"]
+    assert payload["metrics"]["overall_score"] < 100
     assert payload["metrics"]["report_file"] == "审查报告/第20章.md"
 
 
@@ -374,4 +380,4 @@ def test_review_pipeline_main_creates_output_directories(tmp_path):
     finally:
         sys.argv = old_argv
 
-    assert metrics_out.is_file()
+    assert metrics_out.is_file()

+ 1 - 3
webnovel-writer/scripts/review_pipeline.py

@@ -36,9 +36,7 @@ def build_review_artifacts(
 ) -> Dict[str, Any]:
     raw = json.loads(review_results_path.read_text(encoding="utf-8"))
     result = parse_review_output(chapter=chapter, raw=raw)
-    metrics = result.to_metrics_dict()
-    if report_file:
-        metrics["report_file"] = report_file
+    metrics = result.to_metrics_dict(report_file=report_file)
 
     return {
         "chapter": chapter,

+ 42 - 103
webnovel-writer/skills/webnovel-review/SKILL.md

@@ -9,7 +9,7 @@ allowed-tools: Read Grep Write Edit Bash Task AskUserQuestion
 ## 目标
 
 - 解析真实书项目根目录,按统一流程完成章节审查。
-- 调用审查 Agent 生成结构化问题列表、综合评分与审查报告。
+- 调用统一 `reviewer` 生成结构化问题列表与审查报告。
 - 把审查指标写入 `index.db`,并把审查记录写回 `state.json`。
 - 若存在关键问题,明确交给用户决定是否立即返工。
 
@@ -28,27 +28,13 @@ export PROJECT_ROOT="$(python "${SCRIPTS_DIR}/webnovel.py" --project-root "${WOR
 - `PROJECT_ROOT` 必须包含 `.webnovel/state.json`
 - 任一关键目录不存在时立即阻断
 
-### Step 2:记录工作流断点(best-effort)
-
-```bash
-python "${SCRIPTS_DIR}/webnovel.py" --project-root "${PROJECT_ROOT}" workflow start-task --command webnovel-review --chapter {end} || true
-python "${SCRIPTS_DIR}/webnovel.py" --project-root "${PROJECT_ROOT}" workflow start-step --step-id "Step 1" --step-name "解析项目根目录" || true
-python "${SCRIPTS_DIR}/webnovel.py" --project-root "${PROJECT_ROOT}" workflow complete-step --step-id "Step 1" --artifacts '{"project_root_ready":true}' || true
-```
-
-要求:
-- 记录失败只记警告,不阻断主流程
-
-### Step 3:按需加载参考资料
-
-```bash
-python "${SCRIPTS_DIR}/webnovel.py" --project-root "${PROJECT_ROOT}" workflow start-step --step-id "Step 2" --step-name "加载参考" || true
-```
+### Step 2:按需加载参考资料
 
 必读:
 
 ```bash
 cat "${SKILL_ROOT}/../../references/shared/core-constraints.md"
+cat "${SKILL_ROOT}/../../references/review-schema.md"
 ```
 
 按需加载:
@@ -64,111 +50,71 @@ cat "${SKILL_ROOT}/references/pacing-control.md"
 - 先判定 Core 或 Full 审查深度,再加载对应参考
 - 不得在未触发时一次性读完全部资料
 
-```bash
-python "${SCRIPTS_DIR}/webnovel.py" --project-root "${PROJECT_ROOT}" workflow complete-step --step-id "Step 2" --artifacts '{"references_loaded":true}' || true
-```
-
-### Step 4:加载项目状态与待审正文
-
-```bash
-python "${SCRIPTS_DIR}/webnovel.py" --project-root "${PROJECT_ROOT}" workflow start-step --step-id "Step 3" --step-name "加载项目状态" || true
-```
+### Step 3:加载项目状态与待审正文
 
 ```bash
 cat "${PROJECT_ROOT}/.webnovel/state.json"
 ```
 
 要求:
-- 明确当前章节范围与对应正文文件
+- 明确当前章节号与对应正文文件
 - 若缺少正文或状态文件,立即阻断
 
-```bash
-python "${SCRIPTS_DIR}/webnovel.py" --project-root "${PROJECT_ROOT}" workflow complete-step --step-id "Step 3" --artifacts '{"review_input_ready":true}' || true
-```
-
-### Step 5:并行调用检查员并汇总结果
-
-```bash
-python "${SCRIPTS_DIR}/webnovel.py" --project-root "${PROJECT_ROOT}" workflow start-step --step-id "Step 4" --step-name "并行调用检查员" || true
-```
-
-必须通过 `Task` 调用审查子代理,禁止主流程伪造结论。
+### Step 4:调用统一审查 Agent
 
-Core:
-- `consistency-checker`
-- `continuity-checker`
-- `ooc-checker`
-- `reader-pull-checker`
+必须通过 `Task` 调用 `reviewer`,禁止主流程伪造结论。
 
-Full 追加:
-- `high-point-checker`
-- `pacing-checker`
+输入:
+- `chapter`
+- `chapter_file`
+- `project_root`
+- `scripts_dir`
 
-要求:
-- 所有子代理结果返回后,统一汇总 `issues`、`severity`、`overall_score`
-
-```bash
-python "${SCRIPTS_DIR}/webnovel.py" --project-root "${PROJECT_ROOT}" workflow complete-step --step-id "Step 4" --artifacts '{"review_completed":true}' || true
-```
+输出约束:
+- 只输出 JSON
+- 每个 issue 必须有 `evidence`
+- 不输出 `overall_score`
 
-### Step 6:生成审查报告与审查指标 JSON
+中间产物约定:
+- reviewer 原始结果:`${PROJECT_ROOT}/.webnovel/tmp/review_results.json`
+- 落库指标:`${PROJECT_ROOT}/.webnovel/tmp/review_metrics.json`
 
-```bash
-python "${SCRIPTS_DIR}/webnovel.py" --project-root "${PROJECT_ROOT}" workflow start-step --step-id "Step 5" --step-name "生成审查报告" || true
-```
+### Step 5:生成审查报告并落库
 
-报告保存到:`审查报告/第{start}-{end}章审查报告.md`
+报告保存到:`审查报告/第{chapter_num}章审查报告.md`
 
 报告结构:
-- 综合评分
-- 修改优先级
-- 改进建议
-
-审查指标 JSON 必须包含:
-- `start_chapter`
-- `end_chapter`
-- `overall_score`
-- `dimension_scores`
-- `severity_counts`
-- `critical_issues`
-- `report_file`
-- `notes`
+- 总览(问题数 / 阻断数)
+- 阻断问题
+- 其他问题
+- 修复方向
 
-```bash
-python "${SCRIPTS_DIR}/webnovel.py" --project-root "${PROJECT_ROOT}" workflow complete-step --step-id "Step 5" --artifacts '{"report_generated":true}' || true
-```
-
-### Step 7:写入 index.db 与 state.json
+标准文件流:
 
 ```bash
-python "${SCRIPTS_DIR}/webnovel.py" --project-root "${PROJECT_ROOT}" workflow start-step --step-id "Step 6" --step-name "写入审查指标" || true
-```
-
-保存审查指标:
+python -X utf8 "${SCRIPTS_DIR}/webnovel.py" --project-root "${PROJECT_ROOT}" review-pipeline \
+  --chapter {chapter_num} \
+  --review-results "${PROJECT_ROOT}/.webnovel/tmp/review_results.json" \
+  --metrics-out "${PROJECT_ROOT}/.webnovel/tmp/review_metrics.json" \
+  --report-file "审查报告/第{chapter_num}章审查报告.md"
 
-```bash
-python "${SCRIPTS_DIR}/webnovel.py" --project-root "${PROJECT_ROOT}" index save-review-metrics --data '@review_metrics.json'
+python -X utf8 "${SCRIPTS_DIR}/webnovel.py" --project-root "${PROJECT_ROOT}" index save-review-metrics \
+  --data "@${PROJECT_ROOT}/.webnovel/tmp/review_metrics.json"
 ```
 
-写回审查记录:
+要求:
+- `review-pipeline` 生成的 `review_metrics.json` 必须可直接写入 `review_metrics` 表
+- 阻断判断以 reviewer 原始结果中的 `blocking=true` 为准
 
-```bash
-python "${SCRIPTS_DIR}/webnovel.py" --project-root "${PROJECT_ROOT}" update-state -- --add-review "{start}-{end}" "审查报告/第{start}-{end}章审查报告.md"
-```
+### Step 6:写回审查记录并处理阻断
 
-```bash
-python "${SCRIPTS_DIR}/webnovel.py" --project-root "${PROJECT_ROOT}" workflow complete-step --step-id "Step 6" --artifacts '{"review_metrics_saved":true}' || true
-python "${SCRIPTS_DIR}/webnovel.py" --project-root "${PROJECT_ROOT}" workflow start-step --step-id "Step 7" --step-name "写回审查记录" || true
-python "${SCRIPTS_DIR}/webnovel.py" --project-root "${PROJECT_ROOT}" workflow complete-step --step-id "Step 7" --artifacts '{"review_checkpoint_saved":true}' || true
-```
-
-### Step 8:处理关键问题并收尾
+先写回审查记录:
 
 ```bash
-python "${SCRIPTS_DIR}/webnovel.py" --project-root "${PROJECT_ROOT}" workflow start-step --step-id "Step 8" --step-name "处理关键问题并收尾" || true
+python "${SCRIPTS_DIR}/webnovel.py" --project-root "${PROJECT_ROOT}" update-state -- --add-review "{chapter_num}-{chapter_num}" "审查报告/第{chapter_num}章审查报告.md"
 ```
 
-如存在 `critical` 问题,必须使用 `AskUserQuestion` 询问用户:
+如存在任意 `blocking=true` 问题,必须使用 `AskUserQuestion` 询问用户:
 - 立即修复
 - 仅保存报告,稍后处理
 
@@ -179,18 +125,11 @@ python "${SCRIPTS_DIR}/webnovel.py" --project-root "${PROJECT_ROOT}" workflow st
 若用户选择稍后处理:
 - 保留报告与指标记录,结束流程
 
-收尾:
-
-```bash
-python "${SCRIPTS_DIR}/webnovel.py" --project-root "${PROJECT_ROOT}" workflow complete-step --step-id "Step 8" --artifacts '{"ok":true}' || true
-python "${SCRIPTS_DIR}/webnovel.py" --project-root "${PROJECT_ROOT}" workflow complete-task --artifacts '{"ok":true}' || true
-```
-
 ## 成功标准
 
 1. 已解析真实书项目根目录。
-2. 已完成至少 Core 审查深度
+2. 已通过 `reviewer` 输出结构化问题 JSON。
 3. 审查报告已生成。
 4. `review_metrics` 已写入 `index.db`。
 5. 审查记录已写回 `state.json`。
-6. 如存在关键问题,用户已明确选择处理策略。
+6. 如存在阻断问题,用户已明确选择处理策略。