|
|
@@ -1,4 +1,12 @@
|
|
|
-import { readFile, writeFile, listDirectory } from "@/commands/fs"
|
|
|
+import {
|
|
|
+ createDirectory,
|
|
|
+ deleteFile,
|
|
|
+ fileExists,
|
|
|
+ readFile,
|
|
|
+ writeFile,
|
|
|
+ listDirectory,
|
|
|
+} from "@/commands/fs"
|
|
|
+import { computeContextBudget } from "@/lib/context-budget"
|
|
|
import { streamChat } from "@/lib/llm-client"
|
|
|
import type { LlmConfig } from "@/stores/wiki-store"
|
|
|
import { useWikiStore } from "@/stores/wiki-store"
|
|
|
@@ -7,6 +15,7 @@ import i18n from "@/i18n"
|
|
|
import { useActivityStore } from "@/stores/activity-store"
|
|
|
import { useReviewStore, type ReviewItem } from "@/stores/review-store"
|
|
|
import { getFileName, normalizePath } from "@/lib/path-utils"
|
|
|
+import { makeSafeFileSlug } from "@/lib/wiki-filename"
|
|
|
import { checkIngestCache, saveIngestCache } from "@/lib/ingest-cache"
|
|
|
import { sanitizeIngestedFileContent } from "@/lib/ingest-sanitize"
|
|
|
import { mergePageContent, type MergeFn } from "@/lib/page-merge"
|
|
|
@@ -50,6 +59,50 @@ import { buildLanguageDirective } from "@/lib/output-language"
|
|
|
import { detectLanguage } from "@/lib/detect-language"
|
|
|
import { sameScriptFamily } from "@/lib/language-metadata"
|
|
|
|
|
|
+const LONG_SOURCE_MIN_BUDGET = 8_000
|
|
|
+const LONG_SOURCE_MAX_SINGLE_PASS_BUDGET = 300_000
|
|
|
+const LONG_SOURCE_CHUNK_MIN = 12_000
|
|
|
+const LONG_SOURCE_CHUNK_MAX = 60_000
|
|
|
+const LONG_SOURCE_DIGEST_MAX = 15_000
|
|
|
+const LONG_SOURCE_CHUNK_ANALYSIS_MAX = 40_000
|
|
|
+const INGEST_GENERATION_TOKENS_DEFAULT = 8_192
|
|
|
+const INGEST_GENERATION_TOKENS_128K = 16_384
|
|
|
+const INGEST_GENERATION_TOKENS_256K = 24_576
|
|
|
+const INGEST_GENERATION_TOKENS_512K = 32_768
|
|
|
+const REVIEW_STAGE_MIN_SIGNAL_CHARS = 10_000
|
|
|
+const REVIEW_STAGE_MIN_FILE_BLOCKS = 4
|
|
|
+
|
|
|
+interface SourceChunk {
|
|
|
+ id: string
|
|
|
+ index: number
|
|
|
+ total: number
|
|
|
+ headingPath: string
|
|
|
+ overlapBefore: string
|
|
|
+ main: string
|
|
|
+}
|
|
|
+
|
|
|
+interface LongSourcePlan {
|
|
|
+ chunked: boolean
|
|
|
+ analysis: string
|
|
|
+ sourceContext: string
|
|
|
+ checkpointPath?: string
|
|
|
+}
|
|
|
+
|
|
|
+interface LongSourceCheckpoint {
|
|
|
+ version: 1
|
|
|
+ sourceIdentity: string
|
|
|
+ sourceHash: string
|
|
|
+ sourceLength: number
|
|
|
+ sourceBudget: number
|
|
|
+ targetChars: number
|
|
|
+ overlapChars: number
|
|
|
+ chunkTotal: number
|
|
|
+ completedThrough: number
|
|
|
+ globalDigest: string
|
|
|
+ analyses: string[]
|
|
|
+ updatedAt: number
|
|
|
+}
|
|
|
+
|
|
|
// Legacy export kept for backward compatibility with existing diagnostic
|
|
|
// tests. The live pipeline goes through parseFileBlocks() below, which
|
|
|
// handles classes of LLM output this regex silently drops (see H1/H3/H5
|
|
|
@@ -297,6 +350,17 @@ export async function autoIngest(
|
|
|
)
|
|
|
}
|
|
|
|
|
|
+function throwIfIngestAborted(signal: AbortSignal | undefined, activityId?: string): void {
|
|
|
+ if (!signal?.aborted) return
|
|
|
+ if (activityId) {
|
|
|
+ useActivityStore.getState().updateItem(activityId, {
|
|
|
+ status: "error",
|
|
|
+ detail: i18n.t("activity.ingest.cancelled"),
|
|
|
+ })
|
|
|
+ }
|
|
|
+ throw new Error("Ingest cancelled")
|
|
|
+}
|
|
|
+
|
|
|
async function autoIngestImpl(
|
|
|
projectPath: string,
|
|
|
sourcePath: string,
|
|
|
@@ -530,76 +594,119 @@ async function autoIngestImpl(
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- const truncatedContent = enrichedSourceContent.length > 50000
|
|
|
- ? enrichedSourceContent.slice(0, 50000) + "\n\n[...truncated...]"
|
|
|
- : enrichedSourceContent
|
|
|
+ const sourceBaseName = fileName.replace(/\.[^.]+$/, "")
|
|
|
+ const stableContextLength = schema.length + purpose.length + index.length + overview.length
|
|
|
+ const sourceBudget = computeIngestSourceBudget(llmConfig.maxContextSize, stableContextLength)
|
|
|
+ let sourceContext = enrichedSourceContent
|
|
|
+ let precomputedAnalysis = ""
|
|
|
+ let longSourceCheckpointPath: string | undefined
|
|
|
+
|
|
|
+ if (enrichedSourceContent.length > sourceBudget) {
|
|
|
+ const longSourcePlan = await analyzeLongSourceInChunks(
|
|
|
+ pp,
|
|
|
+ llmConfig,
|
|
|
+ purpose,
|
|
|
+ schema,
|
|
|
+ index,
|
|
|
+ fileName,
|
|
|
+ sourceBaseName,
|
|
|
+ folderContext,
|
|
|
+ enrichedSourceContent,
|
|
|
+ sourceBudget,
|
|
|
+ activityId,
|
|
|
+ signal,
|
|
|
+ )
|
|
|
+ if (longSourcePlan.chunked) {
|
|
|
+ sourceContext = longSourcePlan.sourceContext
|
|
|
+ precomputedAnalysis = longSourcePlan.analysis
|
|
|
+ longSourceCheckpointPath = longSourcePlan.checkpointPath
|
|
|
+ }
|
|
|
+ }
|
|
|
|
|
|
// ── Step 1: Analysis ──────────────────────────────────────────
|
|
|
// LLM reads the source and produces a structured analysis:
|
|
|
// key entities, concepts, main arguments, connections to existing wiki, contradictions
|
|
|
- activity.updateItem(activityId, { detail: i18n.t("activity.ingest.analyzingSource") })
|
|
|
-
|
|
|
- let analysis = ""
|
|
|
+ activity.updateItem(activityId, {
|
|
|
+ detail: precomputedAnalysis
|
|
|
+ ? i18n.t("activity.ingest.consolidatingLongSource")
|
|
|
+ : i18n.t("activity.ingest.analyzingSource"),
|
|
|
+ })
|
|
|
|
|
|
- await streamChat(
|
|
|
- llmConfig,
|
|
|
- [
|
|
|
- { role: "system", content: buildAnalysisPrompt(purpose, index, truncatedContent) },
|
|
|
- { role: "user", content: `Analyze this source document:\n\n**File:** ${fileName}${folderContext ? `\n**Folder context:** ${folderContext}` : ""}\n\n---\n\n${truncatedContent}` },
|
|
|
- ],
|
|
|
- {
|
|
|
- onToken: (token) => { analysis += token },
|
|
|
- onDone: () => {},
|
|
|
- onError: (err) => {
|
|
|
- activity.updateItem(activityId, { status: "error", detail: i18n.t("activity.ingest.analysisFailed", { message: err.message }) })
|
|
|
+ let analysis = precomputedAnalysis
|
|
|
+
|
|
|
+ if (!analysis) {
|
|
|
+ const analysisSystem = buildAnalysisPrompt(purpose, index, sourceContext, schema)
|
|
|
+ const analysisUser = `Analyze this source document:\n\n**File:** ${fileName}${folderContext ? `\n**Folder context:** ${folderContext}` : ""}\n\n---\n\n${sourceContext}`
|
|
|
+ await streamChat(
|
|
|
+ llmConfig,
|
|
|
+ [
|
|
|
+ { role: "system", content: analysisSystem },
|
|
|
+ { role: "user", content: analysisUser },
|
|
|
+ ],
|
|
|
+ {
|
|
|
+ onToken: (token) => { analysis += token },
|
|
|
+ onDone: () => {},
|
|
|
+ onError: (err) => {
|
|
|
+ activity.updateItem(activityId, { status: "error", detail: i18n.t("activity.ingest.analysisFailed", { message: err.message }) })
|
|
|
+ },
|
|
|
},
|
|
|
- },
|
|
|
- signal,
|
|
|
- { temperature: 0.1, reasoning: { mode: "off" }, max_tokens: 4096 },
|
|
|
- )
|
|
|
+ signal,
|
|
|
+ {
|
|
|
+ temperature: 0.1,
|
|
|
+ reasoning: { mode: "off" },
|
|
|
+ max_tokens: fitIngestOutputToWindow(
|
|
|
+ llmConfig.maxContextSize,
|
|
|
+ analysisSystem.length + analysisUser.length,
|
|
|
+ computeIngestAnalysisMaxTokens(llmConfig.maxContextSize),
|
|
|
+ ),
|
|
|
+ },
|
|
|
+ )
|
|
|
|
|
|
- // A silent `return []` here would look like success to the queue
|
|
|
- // runner and cause the task to be filter()'d out. Throw instead so
|
|
|
- // processNext's catch-block path (retry / mark failed) engages.
|
|
|
- const analysisActivity = useActivityStore.getState().items.find((i) => i.id === activityId)
|
|
|
- if (analysisActivity?.status === "error") {
|
|
|
- throw new Error(analysisActivity.detail || "Analysis stream failed")
|
|
|
+ // A silent `return []` here would look like success to the queue
|
|
|
+ // runner and cause the task to be filter()'d out. Throw instead so
|
|
|
+ // processNext's catch-block path (retry / mark failed) engages.
|
|
|
+ const analysisActivity = useActivityStore.getState().items.find((i) => i.id === activityId)
|
|
|
+ if (analysisActivity?.status === "error") {
|
|
|
+ throw new Error(analysisActivity.detail || "Analysis stream failed")
|
|
|
+ }
|
|
|
}
|
|
|
|
|
|
+ throwIfIngestAborted(signal, activityId)
|
|
|
+
|
|
|
// ── Step 2: Generation ────────────────────────────────────────
|
|
|
// LLM takes the analysis as context and produces wiki files + review items
|
|
|
activity.updateItem(activityId, { detail: i18n.t("activity.ingest.generatingWikiPages") })
|
|
|
|
|
|
let generation = ""
|
|
|
|
|
|
+ const generationSystem = buildGenerationPrompt(schema, purpose, index, fileName, overview, sourceContext)
|
|
|
+ const generationUser = [
|
|
|
+ `Source document to process: **${fileName}**`,
|
|
|
+ "",
|
|
|
+ "The Stage 1 analysis below is CONTEXT to inform your output. Do NOT echo",
|
|
|
+ "its tables, bullet points, or prose. Your output must be FILE/REVIEW",
|
|
|
+ "blocks as specified in the system prompt — nothing else.",
|
|
|
+ "",
|
|
|
+ "## Stage 1 Analysis (context only — do not repeat)",
|
|
|
+ "",
|
|
|
+ analysis,
|
|
|
+ "",
|
|
|
+ "## Original Source Content",
|
|
|
+ "",
|
|
|
+ sourceContext,
|
|
|
+ "",
|
|
|
+ "---",
|
|
|
+ "",
|
|
|
+ `Now emit the FILE blocks for the wiki files derived from **${fileName}**.`,
|
|
|
+ "Your response MUST begin with `---FILE:` as the very first characters.",
|
|
|
+ "No preamble. No analysis prose. Start immediately.",
|
|
|
+ ].join("\n")
|
|
|
+
|
|
|
await streamChat(
|
|
|
llmConfig,
|
|
|
[
|
|
|
- { role: "system", content: buildGenerationPrompt(schema, purpose, index, fileName, overview, truncatedContent) },
|
|
|
- {
|
|
|
- role: "user",
|
|
|
- content: [
|
|
|
- `Source document to process: **${fileName}**`,
|
|
|
- "",
|
|
|
- "The Stage 1 analysis below is CONTEXT to inform your output. Do NOT echo",
|
|
|
- "its tables, bullet points, or prose. Your output must be FILE/REVIEW",
|
|
|
- "blocks as specified in the system prompt — nothing else.",
|
|
|
- "",
|
|
|
- "## Stage 1 Analysis (context only — do not repeat)",
|
|
|
- "",
|
|
|
- analysis,
|
|
|
- "",
|
|
|
- "## Original Source Content",
|
|
|
- "",
|
|
|
- truncatedContent,
|
|
|
- "",
|
|
|
- "---",
|
|
|
- "",
|
|
|
- `Now emit the FILE blocks for the wiki files derived from **${fileName}**.`,
|
|
|
- "Your response MUST begin with `---FILE:` as the very first characters.",
|
|
|
- "No preamble. No analysis prose. Start immediately.",
|
|
|
- ].join("\n"),
|
|
|
- },
|
|
|
+ { role: "system", content: generationSystem },
|
|
|
+ { role: "user", content: generationUser },
|
|
|
],
|
|
|
{
|
|
|
onToken: (token) => { generation += token },
|
|
|
@@ -609,7 +716,15 @@ async function autoIngestImpl(
|
|
|
},
|
|
|
},
|
|
|
signal,
|
|
|
- { temperature: 0.1, reasoning: { mode: "off" }, max_tokens: 8192 },
|
|
|
+ {
|
|
|
+ temperature: 0.1,
|
|
|
+ reasoning: { mode: "off" },
|
|
|
+ max_tokens: fitIngestOutputToWindow(
|
|
|
+ llmConfig.maxContextSize,
|
|
|
+ generationSystem.length + generationUser.length,
|
|
|
+ computeIngestGenerationMaxTokens(llmConfig.maxContextSize),
|
|
|
+ ),
|
|
|
+ },
|
|
|
)
|
|
|
|
|
|
const generationActivity = useActivityStore.getState().items.find((i) => i.id === activityId)
|
|
|
@@ -617,6 +732,55 @@ async function autoIngestImpl(
|
|
|
throw new Error(generationActivity.detail || "Generation stream failed")
|
|
|
}
|
|
|
|
|
|
+ throwIfIngestAborted(signal, activityId)
|
|
|
+
|
|
|
+ let reviewSuggestionOutput = ""
|
|
|
+ if (!signal?.aborted && shouldRunDedicatedReviewStage(generation)) {
|
|
|
+ let reviewStageHadError = false
|
|
|
+ try {
|
|
|
+ const reviewSystem = buildReviewSuggestionPrompt(
|
|
|
+ purpose,
|
|
|
+ index,
|
|
|
+ fileName,
|
|
|
+ analysis,
|
|
|
+ sourceContext,
|
|
|
+ generation,
|
|
|
+ llmConfig.maxContextSize,
|
|
|
+ )
|
|
|
+ const reviewUser = "Emit only high-value REVIEW blocks for follow-up research or unresolved knowledge gaps. Output nothing if there are none."
|
|
|
+ await streamChat(
|
|
|
+ llmConfig,
|
|
|
+ [
|
|
|
+ { role: "system", content: reviewSystem },
|
|
|
+ { role: "user", content: reviewUser },
|
|
|
+ ],
|
|
|
+ {
|
|
|
+ onToken: (token) => { reviewSuggestionOutput += token },
|
|
|
+ onDone: () => {},
|
|
|
+ onError: (err) => {
|
|
|
+ reviewStageHadError = true
|
|
|
+ console.warn(`[ingest] Review suggestion generation failed for "${fileName}": ${err.message}`)
|
|
|
+ },
|
|
|
+ },
|
|
|
+ signal,
|
|
|
+ {
|
|
|
+ temperature: 0.1,
|
|
|
+ reasoning: { mode: "off" },
|
|
|
+ max_tokens: fitIngestOutputToWindow(
|
|
|
+ llmConfig.maxContextSize,
|
|
|
+ reviewSystem.length + reviewUser.length,
|
|
|
+ computeIngestReviewMaxTokens(llmConfig.maxContextSize),
|
|
|
+ ),
|
|
|
+ },
|
|
|
+ )
|
|
|
+ } catch (err) {
|
|
|
+ throwIfIngestAborted(signal, activityId)
|
|
|
+ console.warn(`[ingest] Review suggestion generation failed for "${fileName}":`, err)
|
|
|
+ }
|
|
|
+ throwIfIngestAborted(signal, activityId)
|
|
|
+ if (reviewStageHadError) reviewSuggestionOutput = ""
|
|
|
+ }
|
|
|
+
|
|
|
// ── Step 3: Write files ───────────────────────────────────────
|
|
|
activity.updateItem(activityId, { detail: i18n.t("activity.ingest.writingFiles") })
|
|
|
const { writtenPaths, warnings: writeWarnings, hardFailures } = await writeFileBlocks(
|
|
|
@@ -639,7 +803,6 @@ async function autoIngestImpl(
|
|
|
}
|
|
|
|
|
|
// Ensure source summary page exists (LLM may not have generated it correctly)
|
|
|
- const sourceBaseName = fileName.replace(/\.[^.]+$/, "")
|
|
|
const sourceSummaryPath = `wiki/sources/${sourceBaseName}.md`
|
|
|
const sourceSummaryFullPath = `${pp}/${sourceSummaryPath}`
|
|
|
const hasSourceSummary = writtenPaths.some((p) => p.startsWith("wiki/sources/"))
|
|
|
@@ -696,7 +859,11 @@ async function autoIngestImpl(
|
|
|
}
|
|
|
|
|
|
// ── Step 4: Parse review items ────────────────────────────────
|
|
|
- const reviewItems = parseReviewBlocks(generation, sp)
|
|
|
+ throwIfIngestAborted(signal, activityId)
|
|
|
+ const reviewItems = [
|
|
|
+ ...parseReviewBlocks(generation, sp),
|
|
|
+ ...parseReviewBlocks(reviewSuggestionOutput, sp),
|
|
|
+ ]
|
|
|
if (reviewItems.length > 0) {
|
|
|
useReviewStore.getState().addItems(reviewItems)
|
|
|
}
|
|
|
@@ -712,6 +879,9 @@ async function autoIngestImpl(
|
|
|
// safe.
|
|
|
if (writtenPaths.length > 0 && hardFailures.length === 0) {
|
|
|
await saveIngestCache(pp, fileName, sourceContent, writtenPaths)
|
|
|
+ if (longSourceCheckpointPath) {
|
|
|
+ await clearLongSourceCheckpoint(longSourceCheckpointPath)
|
|
|
+ }
|
|
|
} else if (hardFailures.length > 0) {
|
|
|
console.warn(
|
|
|
`[ingest] 跳过 "${fileName}" 的缓存保存 — ${hardFailures.length} 个代码块写入失败:${hardFailures.join(", ")}`,
|
|
|
@@ -974,11 +1144,584 @@ function parseReviewBlocks(
|
|
|
return items
|
|
|
}
|
|
|
|
|
|
+function countFileBlocks(text: string): number {
|
|
|
+ return (text.match(/---FILE:\s*[^-]+---/g) ?? []).length
|
|
|
+}
|
|
|
+
|
|
|
+function shouldRunDedicatedReviewStage(generation: string): boolean {
|
|
|
+ return generation.length >= REVIEW_STAGE_MIN_SIGNAL_CHARS
|
|
|
+ || countFileBlocks(generation) >= REVIEW_STAGE_MIN_FILE_BLOCKS
|
|
|
+ || /---REVIEW:\s*[\w-]+\s*\|[\s\S]*$/i.test(generation)
|
|
|
+}
|
|
|
+
|
|
|
+function buildReviewSuggestionPrompt(
|
|
|
+ purpose: string,
|
|
|
+ index: string,
|
|
|
+ sourceIdentity: string,
|
|
|
+ analysis: string,
|
|
|
+ sourceContext: string,
|
|
|
+ generation: string,
|
|
|
+ maxContextSize: number | undefined,
|
|
|
+): string {
|
|
|
+ const { maxCtx } = computeContextBudget(maxContextSize)
|
|
|
+ const sectionCap = Math.max(4_000, Math.floor(maxCtx * 0.15))
|
|
|
+ const indexCap = Math.max(3_000, Math.floor(sectionCap * 0.8))
|
|
|
+ return [
|
|
|
+ "You are identifying high-value follow-up research items for a personal wiki.",
|
|
|
+ "Do not output chain-of-thought, hidden reasoning, or explanatory preamble.",
|
|
|
+ "",
|
|
|
+ languageRule(sourceContext),
|
|
|
+ "",
|
|
|
+ "Your job is NOT to generate wiki pages. The wiki page generation already happened.",
|
|
|
+ "Output only REVIEW blocks for unresolved knowledge gaps that deserve human attention or Deep Research.",
|
|
|
+ "",
|
|
|
+ "Create REVIEW blocks only for genuinely useful follow-up work:",
|
|
|
+ "- missing-page: an important entity/concept is referenced but still lacks a dedicated page",
|
|
|
+ "- suggestion: a research question, source type, or comparison that would materially improve the wiki",
|
|
|
+ "- contradiction: a conflict or tension that requires user judgment",
|
|
|
+ "- duplicate: likely duplicate pages/names that need user review",
|
|
|
+ "",
|
|
|
+ "Prefer 1-5 high-signal reviews. If there is nothing worth reviewing, output nothing.",
|
|
|
+ "For suggestion and missing-page reviews, include a SEARCH line with 2-3 keyword-rich web search queries separated by ` | `.",
|
|
|
+ "Use only these options: OPTIONS: Create Page | Skip",
|
|
|
+ "",
|
|
|
+ "REVIEW block template:",
|
|
|
+ "```",
|
|
|
+ "---REVIEW: suggestion | Precise title---",
|
|
|
+ "Concise description of the gap and why it matters.",
|
|
|
+ "OPTIONS: Create Page | Skip",
|
|
|
+ "PAGES: wiki/page1.md, wiki/page2.md",
|
|
|
+ "SEARCH: query 1 | query 2 | query 3",
|
|
|
+ "---END REVIEW---",
|
|
|
+ "```",
|
|
|
+ "",
|
|
|
+ "Return REVIEW blocks only. Do not output FILE blocks. Do not wrap the response in markdown fences.",
|
|
|
+ "",
|
|
|
+ purpose ? `## Wiki Purpose\n${purpose}` : "",
|
|
|
+ index ? `## Current Wiki Index\n${trimLongText(index, indexCap)}` : "",
|
|
|
+ "",
|
|
|
+ `## Source\n${sourceIdentity}`,
|
|
|
+ "",
|
|
|
+ "## Stage 1 Analysis",
|
|
|
+ trimLongText(analysis, sectionCap),
|
|
|
+ "",
|
|
|
+ "## Source Context",
|
|
|
+ trimLongText(sourceContext, sectionCap),
|
|
|
+ "",
|
|
|
+ "## Generated Wiki Output",
|
|
|
+ trimLongText(generation, sectionCap),
|
|
|
+ ].filter(Boolean).join("\n")
|
|
|
+}
|
|
|
+
|
|
|
+function clampNumber(value: number, min: number, max: number): number {
|
|
|
+ return Math.max(min, Math.min(max, value))
|
|
|
+}
|
|
|
+
|
|
|
+export function computeIngestSourceBudget(
|
|
|
+ maxContextSize: number | undefined,
|
|
|
+ stableContextLength: number,
|
|
|
+ langScale?: number,
|
|
|
+): number {
|
|
|
+ const { maxCtx, responseReserve } = computeContextBudget(maxContextSize, langScale)
|
|
|
+ const stableReserve = Math.min(Math.floor(maxCtx * 0.25), Math.max(12_000, stableContextLength))
|
|
|
+ const instructionReserve = Math.max(12_000, Math.floor(maxCtx * 0.08))
|
|
|
+ const available = maxCtx - responseReserve - stableReserve - instructionReserve
|
|
|
+ const upper = Math.min(LONG_SOURCE_MAX_SINGLE_PASS_BUDGET, Math.max(LONG_SOURCE_MIN_BUDGET, Math.floor(maxCtx * 0.6)))
|
|
|
+ return clampNumber(Math.floor(available), LONG_SOURCE_MIN_BUDGET, upper)
|
|
|
+}
|
|
|
+
|
|
|
+export function computeIngestGenerationMaxTokens(
|
|
|
+ maxContextSize: number | undefined,
|
|
|
+ langScale?: number,
|
|
|
+): number {
|
|
|
+ const { maxCtx } = computeContextBudget(maxContextSize, langScale)
|
|
|
+ if (maxCtx >= 512_000) return INGEST_GENERATION_TOKENS_512K
|
|
|
+ if (maxCtx >= 256_000) return INGEST_GENERATION_TOKENS_256K
|
|
|
+ if (maxCtx >= 128_000) return INGEST_GENERATION_TOKENS_128K
|
|
|
+ return INGEST_GENERATION_TOKENS_DEFAULT
|
|
|
+}
|
|
|
+
|
|
|
+export function computeIngestReviewMaxTokens(
|
|
|
+ maxContextSize: number | undefined,
|
|
|
+ langScale?: number,
|
|
|
+): number {
|
|
|
+ return Math.min(8_192, Math.max(4_096, Math.floor(computeIngestGenerationMaxTokens(maxContextSize, langScale) / 2)))
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * Output-token budget for the intermediate analysis passes (whole-source
|
|
|
+ * analysis and per-chunk long-source analysis). Previously hard-coded to
|
|
|
+ * 4096; now scales off the generation ladder so larger context windows get
|
|
|
+ * a richer analysis, while staying capped well below a full page-generation
|
|
|
+ * pass. Small windows retain the original 4096 floor.
|
|
|
+ */
|
|
|
+export function computeIngestAnalysisMaxTokens(
|
|
|
+ maxContextSize: number | undefined,
|
|
|
+ langScale?: number,
|
|
|
+): number {
|
|
|
+ return Math.min(8_192, Math.max(4_096, Math.floor(computeIngestGenerationMaxTokens(maxContextSize, langScale) / 2)))
|
|
|
+}
|
|
|
+
|
|
|
+/** chars/token the ingest budgeting assumes; mirrors context-budget.ts. */
|
|
|
+const INGEST_CHARS_PER_TOKEN = 4
|
|
|
+/** Smallest output allowance we will still request when the window is nearly
|
|
|
+ * full — below this a response is useless, so we accept a tiny overflow risk
|
|
|
+ * rather than emitting nothing. */
|
|
|
+const INGEST_OUTPUT_TOKEN_FLOOR = 512
|
|
|
+
|
|
|
+/**
|
|
|
+ * Clamp a desired output-token count so that (packed prompt + output) fits the
|
|
|
+ * model's real token window. `desiredTokens` is the ladder value; we only ever
|
|
|
+ * reduce it when the prompt already leaves less room than the ladder wants.
|
|
|
+ *
|
|
|
+ * Language-aware: CJK text is ~2.3x denser, so the same prompt consumes more
|
|
|
+ * real tokens and leaves less room for output. The raw (unscaled) window is
|
|
|
+ * the real token capacity (English-calibrated 4:1); the effective scale
|
|
|
+ * recovers the true chars/token for the active language.
|
|
|
+ */
|
|
|
+export function fitIngestOutputToWindow(
|
|
|
+ maxContextSize: number | undefined,
|
|
|
+ promptChars: number,
|
|
|
+ desiredTokens: number,
|
|
|
+ langScale?: number,
|
|
|
+): number {
|
|
|
+ const rawWindow = computeContextBudget(maxContextSize, 1).maxCtx
|
|
|
+ const scaledWindow = computeContextBudget(maxContextSize, langScale).maxCtx
|
|
|
+ const scale = rawWindow > 0 ? scaledWindow / rawWindow : 1
|
|
|
+ const windowTokens = rawWindow / INGEST_CHARS_PER_TOKEN
|
|
|
+ const inputTokens = promptChars / (INGEST_CHARS_PER_TOKEN * scale)
|
|
|
+ const remaining = Math.floor(windowTokens - inputTokens)
|
|
|
+ return Math.max(INGEST_OUTPUT_TOKEN_FLOOR, Math.min(desiredTokens, remaining))
|
|
|
+}
|
|
|
+
|
|
|
+function splitOversizedBlock(block: string, targetChars: number): string[] {
|
|
|
+ if (block.length <= targetChars * 1.25) return [block]
|
|
|
+
|
|
|
+ const pieces = block.match(/[^.!?。!?\n]+[.!?。!?]?|\n+/g) ?? [block]
|
|
|
+ const out: string[] = []
|
|
|
+ let current = ""
|
|
|
+ for (const piece of pieces) {
|
|
|
+ if (current && current.length + piece.length > targetChars) {
|
|
|
+ out.push(current.trim())
|
|
|
+ current = ""
|
|
|
+ }
|
|
|
+ if (piece.length > targetChars) {
|
|
|
+ for (let i = 0; i < piece.length; i += targetChars) {
|
|
|
+ const slice = piece.slice(i, i + targetChars).trim()
|
|
|
+ if (slice) out.push(slice)
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ current += piece
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if (current.trim()) out.push(current.trim())
|
|
|
+ return out
|
|
|
+}
|
|
|
+
|
|
|
+function semanticBlocks(content: string, targetChars: number): Array<{ text: string; headingPath: string }> {
|
|
|
+ const blocks: Array<{ text: string; headingPath: string }> = []
|
|
|
+ const headingStack: string[] = []
|
|
|
+ let paragraph: string[] = []
|
|
|
+ let paragraphHeading = ""
|
|
|
+
|
|
|
+ const currentHeadingPath = () => headingStack.filter(Boolean).join(" > ")
|
|
|
+ const flushParagraph = () => {
|
|
|
+ const text = paragraph.join("\n").trim()
|
|
|
+ if (text) {
|
|
|
+ for (const piece of splitOversizedBlock(text, targetChars)) {
|
|
|
+ blocks.push({ text: piece, headingPath: paragraphHeading })
|
|
|
+ }
|
|
|
+ }
|
|
|
+ paragraph = []
|
|
|
+ }
|
|
|
+
|
|
|
+ for (const line of content.replace(/\r\n/g, "\n").split("\n")) {
|
|
|
+ const heading = /^(#{1,6})\s+(.+?)\s*$/.exec(line)
|
|
|
+ if (heading) {
|
|
|
+ flushParagraph()
|
|
|
+ const depth = heading[1].length
|
|
|
+ headingStack.length = depth - 1
|
|
|
+ headingStack[depth - 1] = heading[2].trim()
|
|
|
+ blocks.push({ text: line.trim(), headingPath: currentHeadingPath() })
|
|
|
+ paragraphHeading = currentHeadingPath()
|
|
|
+ continue
|
|
|
+ }
|
|
|
+
|
|
|
+ if (line.trim() === "") {
|
|
|
+ flushParagraph()
|
|
|
+ paragraphHeading = currentHeadingPath()
|
|
|
+ continue
|
|
|
+ }
|
|
|
+
|
|
|
+ if (paragraph.length === 0) paragraphHeading = currentHeadingPath()
|
|
|
+ paragraph.push(line)
|
|
|
+ }
|
|
|
+ flushParagraph()
|
|
|
+
|
|
|
+ return blocks
|
|
|
+}
|
|
|
+
|
|
|
+function overlapSuffix(text: string, maxChars: number): string {
|
|
|
+ if (!text || maxChars <= 0) return ""
|
|
|
+ if (text.length <= maxChars) return text
|
|
|
+ const raw = text.slice(-maxChars)
|
|
|
+ const paragraphBreak = raw.search(/\n\s*\n/)
|
|
|
+ if (paragraphBreak > 0 && raw.length - paragraphBreak > maxChars * 0.4) {
|
|
|
+ return raw.slice(paragraphBreak).trim()
|
|
|
+ }
|
|
|
+ const sentenceBreak = raw.search(/[.!?。!?]\s+/)
|
|
|
+ if (sentenceBreak > 0 && raw.length - sentenceBreak > maxChars * 0.4) {
|
|
|
+ return raw.slice(sentenceBreak + 1).trim()
|
|
|
+ }
|
|
|
+ return raw.trim()
|
|
|
+}
|
|
|
+
|
|
|
+export function splitSourceIntoSemanticChunks(
|
|
|
+ content: string,
|
|
|
+ targetChars: number,
|
|
|
+ overlapChars: number,
|
|
|
+): SourceChunk[] {
|
|
|
+ const target = Math.max(1_000, targetChars)
|
|
|
+ const blocks = semanticBlocks(content, target)
|
|
|
+ if (blocks.length === 0) return []
|
|
|
+
|
|
|
+ const rawChunks: Array<{ main: string; headingPath: string }> = []
|
|
|
+ let current: string[] = []
|
|
|
+ let currentLength = 0
|
|
|
+ let currentHeading = blocks[0]?.headingPath ?? ""
|
|
|
+
|
|
|
+ const flush = () => {
|
|
|
+ const main = current.join("\n\n").trim()
|
|
|
+ if (main) rawChunks.push({ main, headingPath: currentHeading })
|
|
|
+ current = []
|
|
|
+ currentLength = 0
|
|
|
+ }
|
|
|
+
|
|
|
+ for (const block of blocks) {
|
|
|
+ const nextLength = currentLength + block.text.length + (current.length > 0 ? 2 : 0)
|
|
|
+ if (current.length > 0 && nextLength > target) {
|
|
|
+ flush()
|
|
|
+ }
|
|
|
+ if (current.length === 0) currentHeading = block.headingPath
|
|
|
+ current.push(block.text)
|
|
|
+ currentLength += block.text.length + (current.length > 1 ? 2 : 0)
|
|
|
+ }
|
|
|
+ flush()
|
|
|
+
|
|
|
+ return rawChunks.map((chunk, idx) => ({
|
|
|
+ id: `chunk-${idx + 1}`,
|
|
|
+ index: idx + 1,
|
|
|
+ total: rawChunks.length,
|
|
|
+ headingPath: chunk.headingPath,
|
|
|
+ overlapBefore: idx > 0 ? overlapSuffix(rawChunks[idx - 1].main, overlapChars) : "",
|
|
|
+ main: chunk.main,
|
|
|
+ }))
|
|
|
+}
|
|
|
+
|
|
|
+function trimLongText(text: string, maxChars: number): string {
|
|
|
+ if (text.length <= maxChars) return text
|
|
|
+ return `${text.slice(0, maxChars).trimEnd()}\n\n[...trimmed for prompt budget...]`
|
|
|
+}
|
|
|
+
|
|
|
+function hashTextHex(text: string): string {
|
|
|
+ let hash = 0xcbf29ce484222325n
|
|
|
+ const prime = 0x100000001b3n
|
|
|
+ for (let i = 0; i < text.length; i++) {
|
|
|
+ hash ^= BigInt(text.charCodeAt(i))
|
|
|
+ hash = BigInt.asUintN(64, hash * prime)
|
|
|
+ }
|
|
|
+ return hash.toString(16).padStart(16, "0")
|
|
|
+}
|
|
|
+
|
|
|
+function longSourceCheckpointPath(
|
|
|
+ projectPath: string,
|
|
|
+ sourceSummarySlug: string,
|
|
|
+ sourceHash: string,
|
|
|
+): string {
|
|
|
+ const safeSlug = makeSafeFileSlug(sourceSummarySlug, "source")
|
|
|
+ return `${normalizePath(projectPath)}/.qmai/ingest-progress/${safeSlug}-${sourceHash}.json`
|
|
|
+}
|
|
|
+
|
|
|
+function isCompatibleLongSourceCheckpoint(
|
|
|
+ checkpoint: LongSourceCheckpoint,
|
|
|
+ params: {
|
|
|
+ sourceIdentity: string
|
|
|
+ sourceHash: string
|
|
|
+ sourceLength: number
|
|
|
+ sourceBudget: number
|
|
|
+ targetChars: number
|
|
|
+ overlapChars: number
|
|
|
+ chunkTotal: number
|
|
|
+ },
|
|
|
+): boolean {
|
|
|
+ return checkpoint.version === 1
|
|
|
+ && checkpoint.sourceIdentity === params.sourceIdentity
|
|
|
+ && checkpoint.sourceHash === params.sourceHash
|
|
|
+ && checkpoint.sourceLength === params.sourceLength
|
|
|
+ && checkpoint.sourceBudget === params.sourceBudget
|
|
|
+ && checkpoint.targetChars === params.targetChars
|
|
|
+ && checkpoint.overlapChars === params.overlapChars
|
|
|
+ && checkpoint.chunkTotal === params.chunkTotal
|
|
|
+ && checkpoint.completedThrough >= 0
|
|
|
+ && checkpoint.completedThrough <= params.chunkTotal
|
|
|
+ && Array.isArray(checkpoint.analyses)
|
|
|
+ && checkpoint.analyses.length === checkpoint.completedThrough
|
|
|
+}
|
|
|
+
|
|
|
+async function loadLongSourceCheckpoint(
|
|
|
+ checkpointPath: string,
|
|
|
+ params: Parameters<typeof isCompatibleLongSourceCheckpoint>[1],
|
|
|
+): Promise<LongSourceCheckpoint | null> {
|
|
|
+ try {
|
|
|
+ const raw = await readFile(checkpointPath)
|
|
|
+ const parsed = JSON.parse(raw) as LongSourceCheckpoint
|
|
|
+ if (!isCompatibleLongSourceCheckpoint(parsed, params)) return null
|
|
|
+ return parsed
|
|
|
+ } catch {
|
|
|
+ return null
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+async function saveLongSourceCheckpoint(
|
|
|
+ checkpointPath: string,
|
|
|
+ checkpoint: LongSourceCheckpoint,
|
|
|
+): Promise<void> {
|
|
|
+ const dir = checkpointPath.split("/").slice(0, -1).join("/")
|
|
|
+ await createDirectory(dir)
|
|
|
+ await writeFile(checkpointPath, JSON.stringify(checkpoint, null, 2))
|
|
|
+}
|
|
|
+
|
|
|
+async function clearLongSourceCheckpoint(checkpointPath: string): Promise<void> {
|
|
|
+ try {
|
|
|
+ if (await fileExists(checkpointPath)) {
|
|
|
+ await deleteFile(checkpointPath)
|
|
|
+ }
|
|
|
+ } catch {
|
|
|
+ // Best-effort cleanup.
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+function extractMarkedSection(raw: string, heading: string): string {
|
|
|
+ const escaped = heading.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")
|
|
|
+ const re = new RegExp(`(?:^|\\n)##\\s+${escaped}\\s*\\n([\\s\\S]*?)(?=\\n##\\s|$)`, "i")
|
|
|
+ return re.exec(raw)?.[1]?.trim() ?? ""
|
|
|
+}
|
|
|
+
|
|
|
+function buildChunkAnalysisSystemPrompt(
|
|
|
+ purpose: string,
|
|
|
+ schema: string,
|
|
|
+ index: string,
|
|
|
+ sourceContent: string,
|
|
|
+): string {
|
|
|
+ return [
|
|
|
+ "You are analyzing a long source document for a personal wiki.",
|
|
|
+ "Do not output chain-of-thought, hidden reasoning, or a thinking transcript.",
|
|
|
+ "Analyze only the current MAIN CHUNK. Use overlap and digest for context only.",
|
|
|
+ "Keep stable names consistent with the existing wiki and prior digest.",
|
|
|
+ "",
|
|
|
+ languageRule(sourceContent),
|
|
|
+ "",
|
|
|
+ "Output exactly two markdown sections:",
|
|
|
+ "",
|
|
|
+ "## Chunk Analysis",
|
|
|
+ "- Concise summary of the main chunk",
|
|
|
+ "- New or updated entities",
|
|
|
+ "- New or updated concepts",
|
|
|
+ "- Any schema-defined page types beyond entity/concept that the main chunk genuinely supports",
|
|
|
+ "- Claims, findings, evidence, contradictions",
|
|
|
+ "- Open questions or research gaps",
|
|
|
+ "",
|
|
|
+ "## Updated Global Digest",
|
|
|
+ "A compact document-level digest that incorporates this chunk and preserves prior cross-chunk context.",
|
|
|
+ "Keep this digest structured under: Summary, Entities, Concepts, Schema-Typed Candidates, Claims, Evidence, Contradictions, Open Questions, Cross-Chunk Relations.",
|
|
|
+ "Use schema-defined types only when the source actually supports them; never invent goals, habits, journal entries, decisions, or similar user-authored records that are not present in the source.",
|
|
|
+ "",
|
|
|
+ "Stable project context follows. It changes rarely and should be treated as background:",
|
|
|
+ purpose ? `## Wiki Purpose\n${purpose}` : "",
|
|
|
+ schema ? `## Wiki Schema\n${schema}` : "",
|
|
|
+ index ? `## Current Wiki Index\n${trimLongText(index, 40_000)}` : "",
|
|
|
+ ].filter(Boolean).join("\n")
|
|
|
+}
|
|
|
+
|
|
|
+function buildChunkAnalysisUserPrompt(
|
|
|
+ sourceIdentity: string,
|
|
|
+ folderContext: string | undefined,
|
|
|
+ chunk: SourceChunk,
|
|
|
+ globalDigest: string,
|
|
|
+): string {
|
|
|
+ return [
|
|
|
+ `Source file: ${sourceIdentity}`,
|
|
|
+ folderContext ? `Folder context: ${folderContext}` : "",
|
|
|
+ `Chunk: ${chunk.index}/${chunk.total}`,
|
|
|
+ chunk.headingPath ? `Heading path: ${chunk.headingPath}` : "",
|
|
|
+ "",
|
|
|
+ "## Current Global Digest",
|
|
|
+ globalDigest || "(No prior digest yet.)",
|
|
|
+ "",
|
|
|
+ chunk.overlapBefore ? "## Previous Overlap Context\n" + chunk.overlapBefore : "",
|
|
|
+ "",
|
|
|
+ "## MAIN CHUNK TO ANALYZE",
|
|
|
+ chunk.main,
|
|
|
+ "",
|
|
|
+ "Return only the two requested sections. Do not repeat overlap-only facts unless the main chunk supports them.",
|
|
|
+ ].filter(Boolean).join("\n")
|
|
|
+}
|
|
|
+
|
|
|
+async function analyzeLongSourceInChunks(
|
|
|
+ projectPath: string,
|
|
|
+ llmConfig: LlmConfig,
|
|
|
+ purpose: string,
|
|
|
+ schema: string,
|
|
|
+ index: string,
|
|
|
+ sourceIdentity: string,
|
|
|
+ sourceSummarySlug: string,
|
|
|
+ folderContext: string | undefined,
|
|
|
+ sourceContent: string,
|
|
|
+ sourceBudget: number,
|
|
|
+ activityId: string,
|
|
|
+ signal?: AbortSignal,
|
|
|
+): Promise<LongSourcePlan> {
|
|
|
+ const targetChars = clampNumber(Math.floor(sourceBudget * 0.55), LONG_SOURCE_CHUNK_MIN, LONG_SOURCE_CHUNK_MAX)
|
|
|
+ const overlapChars = clampNumber(Math.floor(targetChars * 0.08), 800, 3_000)
|
|
|
+ const chunks = splitSourceIntoSemanticChunks(sourceContent, targetChars, overlapChars)
|
|
|
+ if (chunks.length <= 1) {
|
|
|
+ return { chunked: false, analysis: "", sourceContext: sourceContent }
|
|
|
+ }
|
|
|
+
|
|
|
+ const activity = useActivityStore.getState()
|
|
|
+ const systemPrompt = buildChunkAnalysisSystemPrompt(purpose, schema, index, sourceContent)
|
|
|
+ const sourceHash = hashTextHex(sourceContent)
|
|
|
+ const checkpointPath = longSourceCheckpointPath(projectPath, sourceSummarySlug, sourceHash)
|
|
|
+ const checkpointParams = {
|
|
|
+ sourceIdentity,
|
|
|
+ sourceHash,
|
|
|
+ sourceLength: sourceContent.length,
|
|
|
+ sourceBudget,
|
|
|
+ targetChars,
|
|
|
+ overlapChars,
|
|
|
+ chunkTotal: chunks.length,
|
|
|
+ }
|
|
|
+ const checkpoint = await loadLongSourceCheckpoint(checkpointPath, checkpointParams)
|
|
|
+ let globalDigest = checkpoint?.globalDigest ?? ""
|
|
|
+ const analyses: string[] = checkpoint?.analyses ? [...checkpoint.analyses] : []
|
|
|
+ let completedThrough = checkpoint?.completedThrough ?? 0
|
|
|
+
|
|
|
+ if (completedThrough > 0) {
|
|
|
+ activity.updateItem(activityId, {
|
|
|
+ detail: i18n.t("activity.ingest.resumingLongSourceChunk", {
|
|
|
+ current: completedThrough + 1,
|
|
|
+ total: chunks.length,
|
|
|
+ }),
|
|
|
+ })
|
|
|
+ }
|
|
|
+
|
|
|
+ for (const chunk of chunks) {
|
|
|
+ if (chunk.index <= completedThrough) continue
|
|
|
+ throwIfIngestAborted(signal, activityId)
|
|
|
+ activity.updateItem(activityId, {
|
|
|
+ detail: i18n.t("activity.ingest.analyzingLongSourceChunk", {
|
|
|
+ current: chunk.index,
|
|
|
+ total: chunk.total,
|
|
|
+ }),
|
|
|
+ })
|
|
|
+
|
|
|
+ let raw = ""
|
|
|
+ let hadError = false
|
|
|
+ const chunkUser = buildChunkAnalysisUserPrompt(
|
|
|
+ sourceIdentity,
|
|
|
+ folderContext,
|
|
|
+ chunk,
|
|
|
+ trimLongText(globalDigest, LONG_SOURCE_DIGEST_MAX),
|
|
|
+ )
|
|
|
+ await streamChat(
|
|
|
+ llmConfig,
|
|
|
+ [
|
|
|
+ { role: "system", content: systemPrompt },
|
|
|
+ { role: "user", content: chunkUser },
|
|
|
+ ],
|
|
|
+ {
|
|
|
+ onToken: (token) => { raw += token },
|
|
|
+ onDone: () => {},
|
|
|
+ onError: (err) => {
|
|
|
+ hadError = true
|
|
|
+ activity.updateItem(activityId, {
|
|
|
+ status: "error",
|
|
|
+ detail: i18n.t("activity.ingest.chunkAnalysisFailed", { message: err.message }),
|
|
|
+ })
|
|
|
+ },
|
|
|
+ },
|
|
|
+ signal,
|
|
|
+ {
|
|
|
+ temperature: 0.1,
|
|
|
+ reasoning: { mode: "off" },
|
|
|
+ max_tokens: fitIngestOutputToWindow(
|
|
|
+ llmConfig.maxContextSize,
|
|
|
+ systemPrompt.length + chunkUser.length,
|
|
|
+ computeIngestAnalysisMaxTokens(llmConfig.maxContextSize),
|
|
|
+ ),
|
|
|
+ },
|
|
|
+ )
|
|
|
+
|
|
|
+ throwIfIngestAborted(signal, activityId)
|
|
|
+ if (hadError) throw new Error("Chunk analysis stream failed")
|
|
|
+
|
|
|
+ const chunkAnalysis = extractMarkedSection(raw, "Chunk Analysis") || raw.trim()
|
|
|
+ const nextDigest = extractMarkedSection(raw, "Updated Global Digest")
|
|
|
+ analyses.push([
|
|
|
+ `## Chunk ${chunk.index}/${chunk.total}${chunk.headingPath ? ` — ${chunk.headingPath}` : ""}`,
|
|
|
+ trimLongText(chunkAnalysis, LONG_SOURCE_CHUNK_ANALYSIS_MAX),
|
|
|
+ ].join("\n"))
|
|
|
+
|
|
|
+ globalDigest = trimLongText(
|
|
|
+ nextDigest || [globalDigest, chunkAnalysis].filter(Boolean).join("\n\n"),
|
|
|
+ LONG_SOURCE_DIGEST_MAX,
|
|
|
+ )
|
|
|
+ completedThrough = chunk.index
|
|
|
+ await saveLongSourceCheckpoint(checkpointPath, {
|
|
|
+ version: 1,
|
|
|
+ ...checkpointParams,
|
|
|
+ completedThrough,
|
|
|
+ globalDigest,
|
|
|
+ analyses,
|
|
|
+ updatedAt: Date.now(),
|
|
|
+ })
|
|
|
+ }
|
|
|
+
|
|
|
+ const analysis = [
|
|
|
+ "# Consolidated Long-Document Analysis",
|
|
|
+ "",
|
|
|
+ "## Final Global Digest",
|
|
|
+ globalDigest || "(No digest produced.)",
|
|
|
+ "",
|
|
|
+ "## Per-Chunk Analyses",
|
|
|
+ analyses.join("\n\n"),
|
|
|
+ ].join("\n")
|
|
|
+
|
|
|
+ const sourceContext = [
|
|
|
+ `# Long Source Context: ${sourceIdentity}`,
|
|
|
+ "",
|
|
|
+ `The original source was analyzed in ${chunks.length} semantic chunks with paragraph/section boundaries and overlap. Use this consolidated context instead of assuming the raw document ended early.`,
|
|
|
+ "",
|
|
|
+ "## Final Global Digest",
|
|
|
+ globalDigest || "(No digest produced.)",
|
|
|
+ "",
|
|
|
+ "## Chunk Analysis Notes",
|
|
|
+ trimLongText(analyses.join("\n\n"), Math.max(sourceBudget, LONG_SOURCE_CHUNK_ANALYSIS_MAX)),
|
|
|
+ ].join("\n")
|
|
|
+
|
|
|
+ return { chunked: true, analysis, sourceContext, checkpointPath }
|
|
|
+}
|
|
|
+
|
|
|
/**
|
|
|
* Step 1 prompt: AI reads the source and produces a structured analysis.
|
|
|
* This is the "discussion" step — the AI reasons about the source before writing wiki pages.
|
|
|
*/
|
|
|
-export function buildAnalysisPrompt(purpose: string, index: string, sourceContent: string = ""): string {
|
|
|
+export function buildAnalysisPrompt(
|
|
|
+ purpose: string,
|
|
|
+ index: string,
|
|
|
+ sourceContent: string = "",
|
|
|
+ schema: string = "",
|
|
|
+): string {
|
|
|
return [
|
|
|
"You are an expert research analyst. Read the source document and produce a structured analysis.",
|
|
|
"Do not output chain-of-thought, hidden reasoning, or a thinking transcript. Reason internally and write only the concise final analysis.",
|
|
|
@@ -1003,6 +1746,7 @@ export function buildAnalysisPrompt(purpose: string, index: string, sourceConten
|
|
|
"- What are the core claims or results?",
|
|
|
"- What evidence supports them?",
|
|
|
"- How strong is the evidence?",
|
|
|
+ "- Which named subject is each claim about? Do not transfer claims, limits, or evaluations from one entity/model/product/method to another just because they share keywords.",
|
|
|
"",
|
|
|
"## Connections to Existing Wiki",
|
|
|
"- What existing pages does this source relate to?",
|
|
|
@@ -1014,6 +1758,7 @@ export function buildAnalysisPrompt(purpose: string, index: string, sourceConten
|
|
|
"",
|
|
|
"## Recommendations",
|
|
|
"- What wiki pages should be created or updated?",
|
|
|
+ "- If the project schema (below) defines page types beyond entity/concept (e.g. goal, habit, reflection, finding, decision, meeting), and the source genuinely contains matching content, recommend pages of those types — name the type explicitly. Only when the source actually supports it; never invent goals/habits/journal entries that aren't in the source.",
|
|
|
"- What should be emphasized vs. de-emphasized?",
|
|
|
"- Any open questions worth flagging for the user?",
|
|
|
"",
|
|
|
@@ -1021,6 +1766,9 @@ export function buildAnalysisPrompt(purpose: string, index: string, sourceConten
|
|
|
"",
|
|
|
"If a folder context is provided, use it as a hint for categorization — the folder structure often reflects the user's organizational intent (e.g., 'papers/energy' suggests the file is an energy-related paper).",
|
|
|
"",
|
|
|
+ schema
|
|
|
+ ? `## Project Schema (page types available — map source content to schema-defined types when it fits)\n${schema}`
|
|
|
+ : "",
|
|
|
purpose ? `## Wiki Purpose (for context)\n${purpose}` : "",
|
|
|
index ? `## Current Wiki Index (for checking existing content)\n${index}` : "",
|
|
|
].filter(Boolean).join("\n")
|