fix(llm): 修复 CJK 资料包密度与显式思考 max_tokens

资料包/深章用 charsPerTokenForLanguage 换字符配额；显式思考时外发规划后的 max_tokens；自定义卡仅在有值时持久化 maxOutputTokens；补回归测试。

Co-authored-by: Cursor <cursoragent@cursor.com>

src/components/settings/sections/custom-provider-cards.tsx

@@ -112,9 +112,13 @@ export function CustomProviderCards() {
       maxContextSize: normalizeUserLlmContextSize(
         updates.maxContextSize ?? prev.maxContextSize,
       ),
-      maxOutputTokens: normalizeUserLlmMaxOutputTokens(
-        updates.maxOutputTokens ?? prev.maxOutputTokens,
-      ),
+      ...(updates.maxOutputTokens !== undefined || prev.maxOutputTokens !== undefined
+        ? {
+            maxOutputTokens: normalizeUserLlmMaxOutputTokens(
+              updates.maxOutputTokens ?? prev.maxOutputTokens,
+            ),
+          }
+        : {}),
       reasoning: updates.reasoning ?? prev.reasoning,
       functionCallingEnabled: updates.functionCallingEnabled ?? prev.functionCallingEnabled,
       enabled: updates.enabled ?? prev.enabled ?? true,

src/lib/llm-client.ts

@@ -175,26 +175,29 @@ export async function streamChat(
 ): Promise<void> {
   let runtimeConfig = await resolveRuntimeLocalCliConfig(config)
   const preparedMessages = applyGlobalUserMemoryToMessages(messages, requestOverrides)
-  // Apply model-specific context size minimums (e.g. DeepSeek → 1M)
   const configuredWindow = getEffectiveMaxContextSize(runtimeConfig)
   const toolScaffoldTokens = estimateRequestScaffoldTokens(requestOverrides?.tools)
   const outputCap = getEffectiveMaxOutputTokens(runtimeConfig)
+  const thinkingFloorTokens = thinkingMinMaxTokens(runtimeConfig.reasoning ?? { mode: "auto" })
   const runtimeBudget = planLlmRequestBudget({
     maxContextSize: configuredWindow,
     // Without an explicit request the response reserve is only used to size
-    // the INPUT trim; it is not sent as max_tokens (see below).
+    // the INPUT trim; it is not sent as max_tokens unless thinking needs it
+    // (see shouldSendMaxTokens below).
     desiredOutputTokens: requestOverrides?.max_tokens
       ?? Math.floor(configuredWindow * RESPONSE_RESERVE_FRAC),
     scaffoldReserveTokens: toolScaffoldTokens,
     minimumContextTokens: 64,
     maxOutputTokensCap: outputCap,
-    thinkingFloorTokens: thinkingMinMaxTokens(runtimeConfig.reasoning ?? { mode: "auto" }),
+    thinkingFloorTokens,
   })
   let effectiveOutputTokens = runtimeBudget.outputTokens
-  // Only surface max_tokens when the caller asked for one. Inventing a value
-  // would replace the provider's own default with our estimate, capping every
-  // long-form generation that deliberately left it unset.
-  let shouldSendMaxTokens = requestOverrides?.max_tokens !== undefined
+  // Emit max_tokens when the caller asked for one, or when explicit thinking
+  // needs a known output allowance (otherwise OpenAI-compatible paths keep
+  // thinking on against an unknown provider default). auto/off without a
+  // caller value still omits the field so long-form keeps the provider default.
+  let shouldSendMaxTokens =
+    requestOverrides?.max_tokens !== undefined || thinkingFloorTokens > 0
   let budgetedMessages: import("./llm-providers").ChatMessage[]
   try {
     budgetedMessages = trimChatMessagesToTokenBudget(

src/lib/llm-client.usage.spec.ts

@@ -3,7 +3,13 @@ import type { LlmConfig } from "@/stores/wiki-store"
 import { streamChat } from "./llm-client"
 import { estimateChatMessagesTokens } from "./chat-request-budget"
 import type { ChatMessage } from "./llm-providers"
-import { LlmContextBudgetError } from "./context-budget"
+import { thinkingMinMaxTokens } from "./llm-providers"
+import {
+  LlmContextBudgetError,
+  RESPONSE_RESERVE_FRAC,
+  planLlmRequestBudget,
+} from "./context-budget"
+import { normalizeUserLlmMaxOutputTokens } from "./llm-context-size"
 
 const mocks = vi.hoisted(() => ({
   fetch: vi.fn(),
@@ -163,6 +169,55 @@ describe("streamChat usage", () => {
     expect(JSON.parse(String(request.body))).not.toHaveProperty("max_tokens")
   })
 
+  it("reasoning.mode=auto 且调用方未传 max_tokens 时请求体仍省略该字段", async () => {
+    mocks.fetch.mockResolvedValue(new Response([
+      'data: {"choices":[{"delta":{"content":"完成"}}]}',
+      "data: [DONE]",
+      "",
+    ].join("\n"), { status: 200 }))
+
+    await streamChat(
+      { ...config, reasoning: { mode: "auto" } },
+      [{ role: "user", content: "写第一章" }],
+      { onToken: vi.fn(), onDone: vi.fn(), onError: vi.fn() },
+    )
+
+    const request = mocks.fetch.mock.calls[0][1] as RequestInit
+    expect(JSON.parse(String(request.body))).not.toHaveProperty("max_tokens")
+  })
+
+  it("reasoning.mode=high 且调用方未传 max_tokens 时发送预算规划的 max_tokens", async () => {
+    mocks.fetch.mockResolvedValue(new Response([
+      'data: {"choices":[{"delta":{"content":"完成"}}]}',
+      "data: [DONE]",
+      "",
+    ].join("\n"), { status: 200 }))
+
+    const reasoning = { mode: "high" as const }
+    const thinkingFloorTokens = thinkingMinMaxTokens(reasoning)
+    expect(thinkingFloorTokens).toBeGreaterThan(0)
+    const planned = planLlmRequestBudget({
+      maxContextSize: config.maxContextSize,
+      desiredOutputTokens: Math.floor(config.maxContextSize * RESPONSE_RESERVE_FRAC),
+      scaffoldReserveTokens: 0,
+      minimumContextTokens: 64,
+      maxOutputTokensCap: normalizeUserLlmMaxOutputTokens(config.maxOutputTokens),
+      thinkingFloorTokens,
+    })
+
+    await streamChat(
+      { ...config, reasoning },
+      [{ role: "user", content: "写第一章" }],
+      { onToken: vi.fn(), onDone: vi.fn(), onError: vi.fn() },
+    )
+
+    const request = mocks.fetch.mock.calls[0][1] as RequestInit
+    expect(JSON.parse(String(request.body))).toMatchObject({
+      max_tokens: planned.outputTokens,
+    })
+    expect(planned.outputTokens).toBeGreaterThanOrEqual(thinkingFloorTokens)
+  })
+
   it("调用方显式传入的超大 max_tokens 收敛到输出上限", async () => {
     mocks.fetch.mockResolvedValue(new Response([
       'data: {"choices":[{"delta":{"content":"完成"}}]}',

src/lib/novel/context-engine.spec.ts

@@ -1,4 +1,6 @@
-import { describe, expect, it } from "vitest"
+import { afterEach, beforeEach, describe, expect, it } from "vitest"
+import i18n from "@/i18n"
+import { charsPerTokenForLanguage } from "@/lib/context-budget"
 import { annotateChapterOutlineStatus, contextPackToPrompt, trimContextPack, type ContextPack } from "./context-engine"
 
 const basePack: ContextPack = {
@@ -56,6 +58,16 @@ describe("annotateChapterOutlineStatus", () => {
 })
 
 describe("trimContextPack 两级裁剪", () => {
+  // Trim-level cases were authored against the old hardcoded ×4 char quota.
+  // Pin English density so tokenBudget→chars stays comparable; CJK coverage
+  // lives in the language-density nested suite below.
+  beforeEach(async () => {
+    await i18n.changeLanguage("en")
+  })
+  afterEach(async () => {
+    await i18n.changeLanguage("zh")
+  })
+
   const fullPack: ContextPack = {
     task: "生成第10章正文",
     chapterGoal: "主角遭遇反派，爆发冲突",
@@ -111,19 +123,16 @@ describe("trimContextPack 两级裁剪", () => {
   })
 
   it("第二级裁剪：放不下的字段做内容精简", () => {
+    // Minimal pack so the only overflowing field is relatedSettings — forces
+    // the partial-trim path instead of whole-field drops.
     const smallPack: ContextPack = {
-      ...fullPack,
-      searchResults: "",
-      graphSearchResults: "",
-      nextChapterAdvice: "",
+      ...basePack,
+      task: "测试",
       relatedSettings: "世界设定内容。" + "详细描述。".repeat(100),
-      canonRules: "",
-      writingStyle: "",
-      timeline: "",
-      revisionDirectives: "",
     }
     const result = trimContextPack(smallPack, 100)
     expect(result.partiallyTrimmedField).toBeDefined()
+    expect(result.partiallyTrimmedField?.fieldKey).toBe("relatedSettings")
     expect(result.partiallyTrimmedField?.keptChars).toBeLessThan(result.partiallyTrimmedField?.originalChars ?? 0)
   })
 
@@ -172,4 +181,36 @@ describe("trimContextPack 两级裁剪", () => {
     expect(result.prompt).not.toContain("第3章：试炼")
     expect(result.prompt).toContain("生成第10章正文")
   })
+
+  describe("语言密度参与字符配额", () => {
+    afterEach(async () => {
+      await i18n.changeLanguage("zh")
+    })
+
+    it("同 token 预算下 CJK 字符配额约为英文的 1/4，并更早触发裁剪", async () => {
+      const tokenBudget = 500
+      // Fits English (×4 → 2000 chars) but not CJK (×1 → 500 chars).
+      const pack: ContextPack = {
+        ...basePack,
+        task: "测试语言密度",
+        soulDoc: "文".repeat(1_200),
+      }
+
+      expect(charsPerTokenForLanguage("zh")).toBe(charsPerTokenForLanguage("en") / 4)
+
+      await i18n.changeLanguage("zh")
+      expect(charsPerTokenForLanguage()).toBe(1)
+      const cjk = trimContextPack(pack, tokenBudget)
+      expect(
+        cjk.trimmedFields.length + (cjk.partiallyTrimmedField ? 1 : 0),
+      ).toBeGreaterThan(0)
+
+      await i18n.changeLanguage("en")
+      expect(charsPerTokenForLanguage()).toBe(4)
+      const en = trimContextPack(pack, tokenBudget)
+      expect(en.trimmedFields).toHaveLength(0)
+      expect(en.partiallyTrimmedField).toBeUndefined()
+      expect(en.finalChars).toBeGreaterThan(cjk.finalChars)
+    })
+  })
 })

src/lib/novel/context-engine.ts

@@ -1,4 +1,7 @@
-import { resolveContextPackTokenBudget } from "@/lib/context-budget"
+import {
+  charsPerTokenForLanguage,
+  resolveContextPackTokenBudget,
+} from "@/lib/context-budget"
 import { listDirectory, readFile } from "@/commands/fs"
 import i18n from "@/i18n"
 import { searchWiki, tokenizeQuery } from "@/lib/search"
@@ -1198,7 +1201,9 @@ export function trimContextPack(
   const resolvedTokenBudget = tokenBudget && tokenBudget > 0
     ? tokenBudget
     : resolveContextPackTokenBudget({ maxContextSize: options?.maxContextSize })
-  const targetChars = resolvedTokenBudget * 4
+  // Match the token estimator: CJK ≈ 1 char/token, English ≈ 4 chars/token.
+  // A hardcoded ×4 over-admits Chinese after maxContextSize became real tokens.
+  const targetChars = Math.floor(resolvedTokenBudget * charsPerTokenForLanguage())
 
   if (totalChars <= targetChars) {
     for (const { title, content } of fieldData) {

src/lib/novel/deep-chapter-generation.ts

@@ -13,7 +13,10 @@ import {
   isReasoningOnlyResponseError,
   withReasoningDisabled,
 } from "@/lib/reasoning-retry";
-import { planChapterRequestBudget } from "@/lib/context-budget";
+import {
+  charsPerTokenForLanguage,
+  planChapterRequestBudget,
+} from "@/lib/context-budget";
 import {
   getEffectiveMaxOutputTokens,
   thinkingMinMaxTokens,
@@ -150,10 +153,6 @@ const defaultDeps: DeepChapterGenerationDeps = {
 const REPEAT_CHECK_MIN_CHARS = 600;
 const REPEAT_WINDOW_CHARS = 120;
 const REPEAT_HIT_LIMIT = 3;
-/** chars/token approximation used to convert the token budget to characters
- *  for the outline cap (mirrors context-budget.ts / contextPackToPrompt). */
-const DEEP_CHAPTER_CHARS_PER_TOKEN = 4;
-
 function hasUsableChapterExecutionContract(contract: ChapterExecutionContract | null): contract is ChapterExecutionContract {
   if (!contract) return false;
   const hasSceneChecks = contract.sceneSteps.some((step) =>
@@ -698,8 +697,9 @@ export async function runDeepChapterGeneration(
   const generationRequestOverrides: RequestOverrides = {
     max_tokens: chapterGenerationBudget.outputTokens,
   };
-  const totalContextCharBudget =
-    totalContextTokenBudget * DEEP_CHAPTER_CHARS_PER_TOKEN;
+  // Same density as the token estimator / trimContextPack (CJK 1, English 4).
+  const charsPerToken = charsPerTokenForLanguage();
+  const totalContextCharBudget = totalContextTokenBudget * charsPerToken;
   const outlineCharCap = Math.floor(
     totalContextCharBudget * DEEP_CHAPTER_OUTLINE_MAX_FRAC,
   );
@@ -731,7 +731,7 @@ export async function runDeepChapterGeneration(
   const restContextTokenBudget = Math.max(
     DEEP_CHAPTER_REST_TOKEN_FLOOR,
     totalContextTokenBudget -
-      Math.ceil(outlineText.length / DEEP_CHAPTER_CHARS_PER_TOKEN),
+      Math.ceil(outlineText.length / charsPerToken),
   );
   const contextPrompt = [
     previousChaptersAnalysis