QMAI/scripts/calibrate-review-weights.mjs

/**
 * 审稿评分权重校准脚本
 * 
 * 通过黄金标准场景和网格搜索找到最优的维度和严重程度权重组合。
 * 用法: node scripts/calibrate-review-weights.mjs
 */

// ---- 黄金标准场景 ----
// 每个场景定义了：一组审稿问题 + 期望的总分（人工评估的"正确答案"）

const GOLD_STANDARD_SCENARIOS = [
  {
    name: "完美章节（零问题）",
    issues: [],
    expectedTotalScore: 100,
    notes: "无任何问题的章节应得满分",
  },
  {
    name: "轻微时间线错误",
    issues: [
      { severity: "error", type: "timeline", count: 1 },
    ],
    expectedTotalScore: 85,
    notes: "单个 facts 维度错误应扣减适量分数",
  },
  {
    name: "重大角色一致性问题",
    issues: [
      { severity: "error", type: "character_consistency", count: 2 },
      { severity: "warning", type: "character_consistency", count: 2 },
    ],
    expectedTotalScore: 65,
    notes: "多个角色问题应大幅拉低总分，但不至于不及格",
  },
  {
    name: "水文 + 缺钩子",
    issues: [
      { severity: "error", type: "plot", count: 1 },
      { severity: "warning", type: "plot", count: 2 },
      { severity: "error", type: "style", count: 1 },
    ],
    expectedTotalScore: 72,
    notes: "剧情推进问题 + 节奏问题共现",
  },
  {
    name: "多处事实错误 + 轻微人物问题",
    issues: [
      { severity: "error", type: "timeline", count: 2 },
      { severity: "error", type: "foreshadowing", count: 1 },
      { severity: "warning", type: "character_consistency", count: 1 },
      { severity: "info", type: "style", count: 3 },
    ],
    expectedTotalScore: 60,
    notes: "事实一致性严重受损，但不应直接归零",
  },
  {
    name: "全面崩坏（多维度严重错误）",
    issues: [
      { severity: "error", type: "character_consistency", count: 3 },
      { severity: "error", type: "timeline", count: 2 },
      { severity: "error", type: "plot", count: 2 },
      { severity: "error", type: "foreshadowing", count: 1 },
      { severity: "warning", type: "style", count: 4 },
      { severity: "info", type: "style", count: 5 },
    ],
    expectedTotalScore: 35,
    notes: "多维度严重错误，总分应在30-40之间",
  },
  {
    name: "轻微提示（仅 info 级别）",
    issues: [
      { severity: "info", type: "style", count: 3 },
      { severity: "info", type: "plot", count: 1 },
    ],
    expectedTotalScore: 88,
    notes: "仅有 info 级别建议，高分轻微下降",
  },
]

// ---- 权重搜索空间 ----
// 每个维度的权重 + 每种严重度的扣分值

const WEIGHT_RANGES = {
  plot: { min: 0.10, max: 0.25, step: 11 },      // 11 points: 0.10, 0.115, ..., 0.25
  character: { min: 0.10, max: 0.20, step: 9 },   // 9 points
  world: { min: 0.05, max: 0.15, step: 9 },
  pacing: { min: 0.10, max: 0.20, step: 9 },
  facts: { min: 0.20, max: 0.35, step: 13 },
  compliance: { min: 0.10, max: 0.20, step: 9 },
}

const DEDUCTION_RANGES = {
  error: { min: 15, max: 30, step: 16 },    // 16 points: 15, 16, ..., 30
  warning: { min: 8, max: 15, step: 8 },
  info: { min: 3, max: 8, step: 6 },
}

// ---- 默认值（当前实现）----

const DEFAULT_WEIGHTS = {
  plot: 0.20,
  character: 0.15,
  world: 0.10,
  pacing: 0.15,
  facts: 0.25,
  compliance: 0.15,
}

const DEFAULT_DEDUCTIONS = {
  error: 20,
  warning: 10,
  info: 5,
}

// ---- 辅助函数 ----

function range(min, max, steps) {
  const result = []
  const stepSize = (max - min) / (steps - 1)
  for (let i = 0; i < steps; i++) {
    result.push(Math.round((min + i * stepSize) * 1000) / 1000)
  }
  return result
}

function generateWeightCombinations(ranges) {
  const dims = Object.keys(ranges)
  const combos = []
  
  const values = {}
  for (const dim of dims) {
    values[dim] = range(ranges[dim].min, ranges[dim].max, ranges[dim].step)
  }
  
  for (const p of values.plot) {
    for (const c of values.character) {
      for (const w of values.world) {
        for (const pa of values.pacing) {
          for (const f of values.facts) {
            for (const co of values.compliance) {
              const sum = p + c + w + pa + f + co
              if (Math.abs(sum - 1.0) < 0.01) {
                combos.push({ plot: p, character: c, world: w, pacing: pa, facts: f, compliance: co })
              }
            }
          }
        }
      }
    }
  }
  return combos
}

function generateDeductionCombinations(ranges) {
  const combos = []
  const errors = range(ranges.error.min, ranges.error.max, ranges.error.step)
  const warnings = range(ranges.warning.min, ranges.warning.max, ranges.warning.step)
  const infos = range(ranges.info.min, ranges.info.max, ranges.info.step)
  
  for (const e of errors) {
    for (const w of warnings) {
      for (const i of infos) {
        // error > warning > info must hold
        if (e > w && w > i) {
          combos.push({ error: e, warning: w, info: i })
        }
      }
    }
  }
  return combos
}

const TYPE_TO_DIM_MAP = {
  "character_consistency": "character",
  "timeline": "facts",
  "foreshadowing": "facts",
  "plot": "plot",
  "style": "pacing",
  "world": "world",
  "compliance": "compliance",
}

function computeScore(issues, weights, deductions) {
  const dimIssues = {}
  for (const dim of Object.keys(weights)) {
    dimIssues[dim] = []
  }
  
  for (const issue of issues) {
    const dim = TYPE_TO_DIM_MAP[issue.type] || "facts"
    for (let j = 0; j < issue.count; j++) {
      dimIssues[dim].push(issue.severity)
    }
  }
  
  let totalScore = 0
  for (const dim of Object.keys(weights)) {
    const deduction = dimIssues[dim].reduce((sum, sev) => {
      return sum + (deductions[sev] || 5)
    }, 0)
    const dimScore = Math.max(0, 100 - deduction)
    totalScore += dimScore * weights[dim]
  }
  
  return Math.round(totalScore)
}

// ---- 主校准流程 ----

console.log("====== 审稿评分权重校准 ======\n")

console.log(`黄金标准场景数: ${GOLD_STANDARD_SCENARIOS.length}`)

// 搜索权重
console.log("\n[1/2] 搜索最优维度权重...")
const weightCombos = generateWeightCombinations(WEIGHT_RANGES)
console.log(`  候选权重组合: ${weightCombos.length}`)

let bestWeightCombo = null
let bestWeightError = Infinity

for (const combo of weightCombos) {
  let totalError = 0
  for (const scenario of GOLD_STANDARD_SCENARIOS) {
    const score = computeScore(scenario.issues, combo, DEFAULT_DEDUCTIONS)
    totalError += Math.abs(score - scenario.expectedTotalScore)
  }
  if (totalError < bestWeightError) {
    bestWeightError = totalError
    bestWeightCombo = combo
  }
}

// 搜索扣分值
console.log("\n[2/2] 搜索最优扣分值...")
const deductionCombos = generateDeductionCombinations(DEDUCTION_RANGES)
console.log(`  候选扣分量组合: ${deductionCombos.length}`)

let bestDeductionCombo = null
let bestDeductionError = Infinity

for (const combo of deductionCombos) {
  let totalError = 0
  for (const scenario of GOLD_STANDARD_SCENARIOS) {
    const score = computeScore(scenario.issues, bestWeightCombo, combo)
    totalError += Math.abs(score - scenario.expectedTotalScore)
  }
  if (totalError < bestDeductionError) {
    bestDeductionError = totalError
    bestDeductionCombo = combo
  }
}

// ---- 输出结果 ----

console.log("\n====== 校准结果 ======\n")

console.log("📊 最佳维度权重：")
for (const dim of Object.keys(DEFAULT_WEIGHTS)) {
  const defVal = DEFAULT_WEIGHTS[dim]
  const calVal = bestWeightCombo[dim]
  const diff = ((calVal - defVal) / defVal * 100).toFixed(1)
  const arrow = calVal > defVal ? "↑" : calVal < defVal ? "↓" : "→"
  console.log(`  ${dim.padEnd(12)} ${defVal.toFixed(2)} → ${calVal.toFixed(2)}  (${arrow}${Math.abs(diff)}%)`)
}

console.log("\n📊 最佳扣分值：")
for (const sev of Object.keys(DEFAULT_DEDUCTIONS)) {
  const defVal = DEFAULT_DEDUCTIONS[sev]
  const calVal = bestDeductionCombo[sev]
  const diff = ((calVal - defVal) / defVal * 100).toFixed(1)
  const arrow = calVal > defVal ? "↑" : calVal < defVal ? "↓" : "→"
  console.log(`  ${sev.padEnd(12)} ${defVal} → ${calVal}  (${arrow}${Math.abs(diff)}%)`)
}

console.log("\n📊 各场景得分对比：")
console.log("  场景".padEnd(24) + "期望".padEnd(8) + "校准前".padEnd(8) + "校准后".padEnd(8) + "改进")
console.log("  " + "-".repeat(56))

let totalBefore = 0
let totalAfter = 0
for (const scenario of GOLD_STANDARD_SCENARIOS) {
  const before = computeScore(scenario.issues, DEFAULT_WEIGHTS, DEFAULT_DEDUCTIONS)
  const after = computeScore(scenario.issues, bestWeightCombo, bestDeductionCombo)
  totalBefore += Math.abs(before - scenario.expectedTotalScore)
  totalAfter += Math.abs(after - scenario.expectedTotalScore)
  const improvement = (Math.abs(before - scenario.expectedTotalScore) - Math.abs(after - scenario.expectedTotalScore)).toFixed(1)
  const arrow = improvement > 0 ? "✅" : improvement < 0 ? "❌" : "➡️"
  console.log(`  ${scenario.name.padEnd(22)} ${String(scenario.expectedTotalScore).padEnd(8)} ${String(before).padEnd(8)} ${String(after).padEnd(8)} ${arrow} ${improvement}`)
}

console.log(`\n  总绝对误差: 校准前 ${totalBefore} → 校准后 ${totalAfter} (降低 ${(totalBefore - totalAfter).toFixed(1)})`)

// ---- 归一化权重 ----
const weightSum = Object.values(bestWeightCombo).reduce((a, b) => a + b, 0)
for (const dim of Object.keys(bestWeightCombo)) {
  bestWeightCombo[dim] = Math.round(bestWeightCombo[dim] / weightSum * 1000) / 1000
}

// ---- 输出推荐配置 ----

console.log("\n====== 推荐配置（可直接用于 ReviewScoringOptions）======")

console.log("\ndimensionWeights: {")
for (const dim of Object.keys(bestWeightCombo)) {
  console.log(`  ${dim}: ${bestWeightCombo[dim].toFixed(3)},`)
}
console.log("}")

console.log("\nseverityDeductions: {")
for (const sev of Object.keys(bestDeductionCombo)) {
  console.log(`  ${sev}: ${bestDeductionCombo[sev]},`)
}
console.log("}")

console.log("\n✅ 校准完成！")