perf(maintenance): 优化 dedup 扫描与合并 I/O 及进度展示

消除扫描时 summary 双倍读盘，并行读取实体/概念页与 wiki 页
合并阶段并行备份与 rewrite 写入，并展示读盘/LLM/写盘进度
扫描展示读盘与模型分析两阶段；重复候选按 high 置信度优先排序

Co-authored-by: Cursor <cursoragent@cursor.com>

src/components/settings/sections/maintenance-section.tsx

@@ -23,7 +23,7 @@ import { useWikiStore, type ProviderConfigs } from "@/stores/wiki-store"
 import { hasUsableLlm } from "@/lib/has-usable-llm"
 import { normalizePath } from "@/lib/path-utils"
 import { resolveDefaultModel, resolveModelConfig } from "@/lib/novel/model-resolver"
-import { loadAllEntitySummaries, runDuplicateDetection } from "@/lib/dedup-runner"
+import { runDuplicateDetection, type DedupScanStage } from "@/lib/dedup-runner"
 import { addNotDuplicate } from "@/lib/dedup-storage"
 import {
   loadDedupScanCache,
@@ -36,14 +36,34 @@ import {
   cancelTask,
   retryTask,
   getQueue,
+  getMergeProgress,
   groupKey,
   ensureQueueActive,
   onDedupMergeComplete,
   type DedupTask,
 } from "@/lib/dedup-queue"
+import type { DedupMergeStage } from "@/lib/dedup-runner"
 import type { WikiProject } from "@/types/wiki"
 import type { DuplicateGroup } from "@/lib/dedup"
 
+function confidenceRank(confidence: DuplicateGroup["confidence"]): number {
+  switch (confidence) {
+    case "high":
+      return 0
+    case "medium":
+      return 1
+    case "low":
+      return 2
+  }
+}
+
+function sortGroupEntriesByConfidence(entries: GroupUiEntry[]): GroupUiEntry[] {
+  return [...entries].sort(
+    (a, b) =>
+      confidenceRank(a.group.confidence) - confidenceRank(b.group.confidence),
+  )
+}
+
 interface GroupUiEntry {
   group: DuplicateGroup
   canonicalSlug: string
@@ -162,6 +182,7 @@ export function MaintenanceSection() {
 
   const [dedupModelId, setDedupModelId] = useState("")
   const [isScanning, setIsScanning] = useState(false)
+  const [scanStage, setScanStage] = useState<DedupScanStage | null>(null)
   const [scanState, setScanState] = useState<MaintenanceScanState>(sharedScanState)
   const [localScanState, setLocalScanState] = useState<MaintenanceScanState | null>(null)
 
@@ -249,23 +270,34 @@ export function MaintenanceSection() {
   // that completed while the user was on a different settings tab).
   // Same pattern activity-panel uses for ingest-queue.
   const [tasks, setTasks] = useState<readonly DedupTask[]>([])
+  const [mergeProgress, setMergeProgress] = useState<{
+    taskId: string
+    stage: DedupMergeStage
+  } | null>(null)
   const [enqueueingKey, setEnqueueingKey] = useState<string | null>(null)
   const [mergeErrors, setMergeErrors] = useState<Record<string, string>>({})
 
   useEffect(() => {
     if (!project) {
       setTasks([])
+      setMergeProgress(null)
       return
     }
     let cancelled = false
     void ensureQueueActive(project.id, project.path)
       .then(() => {
-        if (!cancelled) setTasks([...getQueue()])
+        if (!cancelled) {
+          setTasks([...getQueue()])
+          setMergeProgress(getMergeProgress())
+        }
       })
       .catch((err) => {
         console.error("[Maintenance] ensureQueueActive failed:", err)
       })
-    const id = setInterval(() => setTasks([...getQueue()]), 500)
+    const id = setInterval(() => {
+      setTasks([...getQueue()])
+      setMergeProgress(getMergeProgress())
+    }, 500)
     return () => {
       cancelled = true
       clearInterval(id)
@@ -359,6 +391,7 @@ export function MaintenanceSection() {
     }
 
     setIsScanning(true)
+    setScanStage("loading")
     applyScanState({
       projectId,
       projectPath,
@@ -369,20 +402,11 @@ export function MaintenanceSection() {
       scannedPageCount: null,
     })
     try {
-      const summaries = await loadAllEntitySummaries(projectPath)
-      if (summaries.length < 2) {
-        applyScanState({
-          projectId,
-          projectPath,
-          scanning: false,
-          groups: [],
-          scanCompleted: true,
-          scannedPageCount: summaries.length,
-        })
-        return
-      }
-
-      const detected = await runDuplicateDetection(projectPath, effectiveDedupConfig)
+      const { groups: detected, scannedPageCount } = await runDuplicateDetection(
+        projectPath,
+        effectiveDedupConfig,
+        { onProgress: setScanStage },
+      )
       applyScanState({
         projectId,
         projectPath,
@@ -393,7 +417,7 @@ export function MaintenanceSection() {
           skipped: false,
         })),
         scanCompleted: true,
-        scannedPageCount: summaries.length,
+        scannedPageCount,
       })
     } catch (err) {
       applyScanState({
@@ -406,6 +430,7 @@ export function MaintenanceSection() {
       })
     } finally {
       setIsScanning(false)
+      setScanStage(null)
     }
   }, [project, effectiveDedupConfig, providerConfigs, t, applyScanState])
 
@@ -484,7 +509,7 @@ export function MaintenanceSection() {
   )
 
   const visibleGroups = useMemo(
-    () => groups.filter((entry) => !entry.skipped),
+    () => sortGroupEntriesByConfidence(groups.filter((entry) => !entry.skipped)),
     [groups],
   )
 
@@ -598,9 +623,18 @@ export function MaintenanceSection() {
           <div className="flex items-start gap-1.5 rounded border border-border/60 bg-background/80 px-2 py-1.5 text-xs text-muted-foreground">
             <Loader2 className="mt-0.5 h-3.5 w-3.5 shrink-0 animate-spin" />
             <div>
-              {t("settings.sections.maintenance.dedup.scanningHint", {
-                defaultValue: "正在扫描实体 / 概念页面并调用模型分析，可能需要一会儿…",
-              })}
+              {scanStage === "loading"
+                ? t("settings.sections.maintenance.dedup.scanStageLoading", {
+                    defaultValue: "正在读取实体 / 概念页面…",
+                  })
+                : scanStage === "detecting"
+                  ? t("settings.sections.maintenance.dedup.scanStageDetecting", {
+                      defaultValue: "正在调用模型分析…",
+                    })
+                  : t("settings.sections.maintenance.dedup.scanningHint", {
+                      defaultValue:
+                        "正在扫描实体 / 概念页面并调用模型分析，可能需要一会儿…",
+                    })}
             </div>
           </div>
         )}
@@ -651,6 +685,7 @@ export function MaintenanceSection() {
       <QueueOrphanList
         tasks={tasks}
         groups={groups}
+        mergeProgress={mergeProgress}
         onCancel={(id) => void handleCancel(id)}
         onRetry={(id) => void handleRetry(id)}
         pendingPositionByTaskId={pendingPositionByTaskId}
@@ -665,6 +700,7 @@ export function MaintenanceSection() {
             key={entry.group.slugs.join(",")}
             entry={entry}
             task={task}
+            mergeProgress={mergeProgress}
             enqueueing={enqueueingKey === entryKey}
             mergeError={mergeErrors[entryKey] ?? null}
             pendingPosition={
@@ -689,6 +725,7 @@ export function MaintenanceSection() {
 interface QueueOrphanListProps {
   tasks: readonly DedupTask[]
   groups: GroupUiEntry[]
+  mergeProgress: { taskId: string; stage: DedupMergeStage } | null
   onCancel: (taskId: string) => void
   onRetry: (taskId: string) => void
   pendingPositionByTaskId: Map<string, number>
@@ -704,6 +741,7 @@ interface QueueOrphanListProps {
 function QueueOrphanList({
   tasks,
   groups,
+  mergeProgress,
   onCancel,
   onRetry,
   pendingPositionByTaskId,
@@ -744,6 +782,9 @@ function QueueOrphanList({
             <TaskStatusChip
               task={task}
               pendingPosition={pendingPositionByTaskId.get(task.id) ?? 0}
+              mergeStage={
+                mergeProgress?.taskId === task.id ? mergeProgress.stage : null
+              }
             />
             {task.status === "failed" && (
               <Button
@@ -778,17 +819,42 @@ function QueueOrphanList({
 interface ChipProps {
   task: DedupTask
   pendingPosition: number
+  mergeStage?: DedupMergeStage | null
 }
 
-function TaskStatusChip({ task, pendingPosition }: ChipProps) {
+function mergeStageLabel(
+  stage: DedupMergeStage,
+  t: ReturnType<typeof useTranslation>["t"],
+): string {
+  switch (stage) {
+    case "loading":
+      return t("settings.sections.maintenance.dedup.mergeStageLoading", {
+        defaultValue: "正在读取 wiki…",
+      })
+    case "merging":
+      return t("settings.sections.maintenance.dedup.mergeStageMerging", {
+        defaultValue: "正在合并内容（LLM）…",
+      })
+    case "writing":
+      return t("settings.sections.maintenance.dedup.mergeStageWriting", {
+        defaultValue: "正在写入文件…",
+      })
+  }
+}
+
+function TaskStatusChip({ task, pendingPosition, mergeStage }: ChipProps) {
   const { t } = useTranslation()
   if (task.status === "processing") {
+    const label =
+      mergeStage != null
+        ? mergeStageLabel(mergeStage, t)
+        : t("settings.sections.maintenance.dedup.merging", {
+            defaultValue: "合并中...",
+          })
     return (
       <span className="inline-flex items-center gap-1 rounded bg-amber-500/15 px-1.5 py-0.5 text-[10px] font-semibold uppercase text-amber-700 dark:text-amber-400">
         <Loader2 className="h-3 w-3 animate-spin" />
-        {t("settings.sections.maintenance.dedup.merging", {
-          defaultValue: "合并中...",
-        })}
+        {label}
       </span>
     )
   }
@@ -828,6 +894,7 @@ function TaskStatusChip({ task, pendingPosition }: ChipProps) {
 interface CardProps {
   entry: GroupUiEntry
   task: DedupTask | undefined
+  mergeProgress: { taskId: string; stage: DedupMergeStage } | null
   enqueueing: boolean
   mergeError: string | null
   pendingPosition: number
@@ -841,6 +908,7 @@ interface CardProps {
 function DuplicateGroupCard({
   entry,
   task,
+  mergeProgress,
   enqueueing,
   mergeError,
   pendingPosition,
@@ -887,7 +955,13 @@ function DuplicateGroupCard({
         )}
         {task && !finished && (
           <span className="ml-auto">
-            <TaskStatusChip task={task} pendingPosition={pendingPosition} />
+            <TaskStatusChip
+              task={task}
+              pendingPosition={pendingPosition}
+              mergeStage={
+                mergeProgress?.taskId === task.id ? mergeProgress.stage : null
+              }
+            />
           </span>
         )}
       </div>

src/i18n/en.json

@@ -1042,6 +1042,8 @@
           "noModel": "Add and enable a model under Settings → LLM first.",
           "selectModel": "Select a detection model first.",
           "scanningHint": "Scanning entity / concept pages and analyzing with the model. This may take a while…",
+          "scanStageLoading": "Reading entity / concept pages…",
+          "scanStageDetecting": "Analyzing with the model…",
           "groupsFound": "Found {{count}} duplicate candidate groups. Review them below before merging.",
           "insufficientPages": "At least 2 entity / concept pages are required to detect duplicates. Currently only {{count}}.",
           "noneFound": "No duplicate groups found. The knowledge base is clean.",
@@ -1050,6 +1052,9 @@
           "mergeButton": "Merge into {{slug}}",
           "enqueueing": "Queueing...",
           "merging": "Merging...",
+          "mergeStageLoading": "Reading wiki…",
+          "mergeStageMerging": "Merging content (LLM)…",
+          "mergeStageWriting": "Writing files…",
           "queued": "Queued",
           "queuedAhead": "Queued ({{n}} ahead)",
           "failed": "Failed ({{retries}}/3)",

src/i18n/zh.json

@@ -763,6 +763,8 @@
           "noModel": "请先在「设置 → 大语言模型」中添加并启用一个模型。",
           "selectModel": "请先选择检测模型。",
           "scanningHint": "正在扫描实体 / 概念页面并调用模型分析，可能需要一会儿…",
+          "scanStageLoading": "正在读取实体 / 概念页面…",
+          "scanStageDetecting": "正在调用模型分析…",
           "groupsFound": "发现 {{count}} 组重复候选，请在下方确认是否合并。",
           "insufficientPages": "至少需要 2 个实体 / 概念页面才能检测重复，当前只有 {{count}} 个。",
           "noneFound": "未发现重复分组，当前资料库很干净。",
@@ -771,6 +773,9 @@
           "mergeButton": "合并到 {{slug}}",
           "enqueueing": "加入队列...",
           "merging": "合并中...",
+          "mergeStageLoading": "正在读取 wiki…",
+          "mergeStageMerging": "正在合并内容（LLM）…",
+          "mergeStageWriting": "正在写入文件…",
           "queued": "已排队",
           "queuedAhead": "已排队（前方还有 {{n}} 项）",
           "failed": "失败（{{retries}}/3）",

src/lib/dedup-queue.ts

@@ -21,7 +21,7 @@ import { normalizePath } from "@/lib/path-utils"
 import { getProjectPathById } from "@/lib/project-identity"
 import { hasUsableLlm } from "@/lib/has-usable-llm"
 import { resolveDefaultModel, resolveModelConfig } from "@/lib/novel/model-resolver"
-import { executeMerge } from "@/lib/dedup-runner"
+import { executeMerge, type DedupMergeStage } from "@/lib/dedup-runner"
 import type { DuplicateGroup } from "@/lib/dedup"
 
 // ── Types ─────────────────────────────────────────────────────────────────
@@ -45,6 +45,7 @@ let processing = false
 let currentProjectId = ""
 let currentProjectPath = ""
 let currentAbortController: AbortController | null = null
+let currentMergeProgress: { taskId: string; stage: DedupMergeStage } | null = null
 
 type MergeCompleteListener = (task: DedupTask) => void
 const mergeCompleteListeners = new Set<MergeCompleteListener>()
@@ -213,6 +214,7 @@ export async function cancelTask(taskId: string): Promise<void> {
       currentAbortController = null
     }
     processing = false
+    currentMergeProgress = null
   }
 
   queue = queue.filter((t) => t.id !== taskId)
@@ -224,6 +226,11 @@ export function getQueue(): readonly DedupTask[] {
   return queue
 }
 
+/** In-memory merge stage for the currently processing task (not persisted). */
+export function getMergeProgress(): { taskId: string; stage: DedupMergeStage } | null {
+  return currentMergeProgress
+}
+
 export function getQueueSummary(): {
   pending: number
   processing: number
@@ -252,6 +259,7 @@ export function clearQueueState(): void {
   currentProjectId = ""
   currentProjectPath = ""
   currentAbortController = null
+  currentMergeProgress = null
 }
 
 /**
@@ -269,6 +277,7 @@ export async function pauseQueue(): Promise<void> {
     currentAbortController = null
   }
   processing = false
+  currentMergeProgress = null
 
   for (const task of queue) {
     if (task.status === "processing") {
@@ -382,6 +391,7 @@ async function processNext(projectId: string): Promise<void> {
     next.status = "failed"
     next.error = "LLM 未配置，请在设置中配置大模型提供方"
     processing = false
+    currentMergeProgress = null
     await saveQueue(pp)
     return
   }
@@ -391,14 +401,19 @@ async function processNext(projectId: string): Promise<void> {
   )
 
   currentAbortController = new AbortController()
+  currentMergeProgress = { taskId: next.id, stage: "loading" }
 
   try {
     await executeMerge(pp, next.group, next.canonicalSlug, llmConfig, {
       signal: currentAbortController.signal,
+      onProgress: (stage) => {
+        currentMergeProgress = { taskId: next.id, stage }
+      },
     })
     if (currentProjectId !== projectId) return
 
     currentAbortController = null
+    currentMergeProgress = null
     const completedTask = { ...next }
     queue = queue.filter((t) => t.id !== next.id)
     await saveQueue(pp)
@@ -410,6 +425,7 @@ async function processNext(projectId: string): Promise<void> {
   } catch (err) {
     if (currentProjectId !== projectId) return
     currentAbortController = null
+    currentMergeProgress = null
     const message = err instanceof Error ? err.message : String(err)
     next.retryCount++
     next.error = message

src/lib/dedup-runner.spec.ts

@@ -0,0 +1,242 @@
+import { beforeEach, expect, test, vi } from "vitest"
+import type { FileNode } from "@/types/wiki"
+
+vi.mock("@/commands/fs", () => ({
+  listDirectory: vi.fn(),
+  readFile: vi.fn(),
+  writeFile: vi.fn(),
+  deleteFile: vi.fn(),
+  fileExists: vi.fn(),
+}))
+
+vi.mock("@/lib/llm-client", () => ({
+  streamChat: vi.fn(),
+}))
+
+import { listDirectory, readFile, fileExists } from "@/commands/fs"
+import { streamChat } from "@/lib/llm-client"
+import {
+  loadAllEntitySummaries,
+  loadAllWikiPages,
+  mapWithConcurrency,
+  runDuplicateDetection,
+} from "./dedup-runner"
+import type { EntitySummary } from "./dedup"
+
+const mockedListDirectory = vi.mocked(listDirectory)
+const mockedReadFile = vi.mocked(readFile)
+const mockedFileExists = vi.mocked(fileExists)
+const mockedStreamChat = vi.mocked(streamChat)
+
+function entityMarkdown(title: string): string {
+  return `---
+type: entity
+title: ${title}
+tags: []
+---
+# ${title}
+`
+}
+
+function wikiTree(projectPath: string, pageCount: number): FileNode[] {
+  const children: FileNode[] = []
+  for (let i = 0; i < pageCount; i++) {
+    children.push({
+      name: `page-${i}.md`,
+      path: `${projectPath}/wiki/entities/page-${i}.md`,
+      is_dir: false,
+    })
+  }
+  return [
+    {
+      name: "wiki",
+      path: `${projectPath}/wiki`,
+      is_dir: true,
+      children: [
+        {
+          name: "entities",
+          path: `${projectPath}/wiki/entities`,
+          is_dir: true,
+          children,
+        },
+      ],
+    },
+  ]
+}
+
+beforeEach(() => {
+  vi.clearAllMocks()
+  mockedFileExists.mockResolvedValue(false)
+  mockedStreamChat.mockImplementation(async (_config, _messages, callbacks) => {
+    callbacks.onToken('{"groups": []}')
+    callbacks.onDone()
+  })
+})
+
+test("mapWithConcurrency respects concurrency limit", async () => {
+  let inFlight = 0
+  let maxInFlight = 0
+
+  const results = await mapWithConcurrency(
+    [1, 2, 3, 4, 5, 6],
+    2,
+    async (n) => {
+      inFlight++
+      maxInFlight = Math.max(maxInFlight, inFlight)
+      await new Promise((resolve) => setTimeout(resolve, 20))
+      inFlight--
+      return n * 2
+    },
+  )
+
+  expect(maxInFlight).toBeLessThanOrEqual(2)
+  expect(results.sort((a, b) => a - b)).toEqual([2, 4, 6, 8, 10, 12])
+})
+
+test("mapWithConcurrency omits null and undefined results", async () => {
+  const results = await mapWithConcurrency(["a", "b", "c"], 3, async (item) => {
+    if (item === "b") return null
+    if (item === "c") return undefined
+    return item.toUpperCase()
+  })
+
+  expect(results).toEqual(["A"])
+})
+
+test("mapWithConcurrency returns empty array for empty input", async () => {
+  const fn = vi.fn(async () => "x")
+  const results = await mapWithConcurrency([], 4, fn)
+  expect(results).toEqual([])
+  expect(fn).not.toHaveBeenCalled()
+})
+
+test("loadAllWikiPages reads wiki markdown files in parallel", async () => {
+  mockedListDirectory.mockResolvedValue(wikiTree("/Project", 5))
+  mockedReadFile.mockImplementation(async (path) => `content:${path}`)
+
+  const pages = await loadAllWikiPages("/Project")
+
+  expect(pages).toHaveLength(5)
+  expect(mockedReadFile).toHaveBeenCalledTimes(5)
+  for (const page of pages) {
+    expect(page.path).toMatch(/^wiki\/entities\/page-\d+\.md$/)
+    expect(page.content).toContain("content:")
+  }
+})
+
+test("loadAllWikiPages skips unreadable files", async () => {
+  mockedListDirectory.mockResolvedValue(wikiTree("/Project", 3))
+  mockedReadFile.mockImplementation(async (path) => {
+    if (path.endsWith("page-1.md")) {
+      throw new Error("permission denied")
+    }
+    return `ok:${path}`
+  })
+
+  const pages = await loadAllWikiPages("/Project")
+
+  expect(pages).toHaveLength(2)
+  expect(pages.every((p) => !p.path.endsWith("page-1.md"))).toBe(true)
+})
+
+test("loadAllEntitySummaries reads entity and concept pages in parallel", async () => {
+  mockedListDirectory.mockResolvedValue(wikiTree("/Project", 4))
+  mockedReadFile.mockImplementation(async (path) => {
+    const match = path.match(/page-(\d+)\.md$/)
+    const idx = match?.[1] ?? "0"
+    return entityMarkdown(`Page ${idx}`)
+  })
+
+  const summaries = await loadAllEntitySummaries("/Project")
+
+  expect(summaries).toHaveLength(4)
+  expect(mockedReadFile).toHaveBeenCalledTimes(4)
+  expect(summaries.every((s) => s.type === "entity")).toBe(true)
+})
+
+test("loadAllEntitySummaries skips pages without frontmatter", async () => {
+  mockedListDirectory.mockResolvedValue(wikiTree("/Project", 2))
+  mockedReadFile.mockImplementation(async (path) => {
+    if (path.endsWith("page-0.md")) return entityMarkdown("Valid")
+    return "# no frontmatter\n"
+  })
+
+  const summaries = await loadAllEntitySummaries("/Project")
+
+  expect(summaries).toHaveLength(1)
+  expect(summaries[0]?.slug).toBe("page-0")
+})
+
+test("runDuplicateDetection returns scannedPageCount and skips disk when summaries provided", async () => {
+  const summaries: EntitySummary[] = [
+    {
+      slug: "alpha",
+      path: "wiki/entities/alpha.md",
+      type: "entity",
+      title: "Alpha",
+      tags: [],
+    },
+    {
+      slug: "beta",
+      path: "wiki/entities/beta.md",
+      type: "entity",
+      title: "Beta",
+      tags: [],
+    },
+  ]
+
+  const result = await runDuplicateDetection(
+    "/Project",
+    { provider: "openai", model: "gpt-4", apiKey: "test" },
+    { summaries },
+  )
+
+  expect(mockedListDirectory).not.toHaveBeenCalled()
+  expect(mockedReadFile).not.toHaveBeenCalled()
+  expect(result.scannedPageCount).toBe(2)
+  expect(result.groups).toEqual([])
+  expect(mockedStreamChat).toHaveBeenCalledTimes(1)
+})
+
+test("runDuplicateDetection short-circuits when fewer than two summaries", async () => {
+  const summaries: EntitySummary[] = [
+    {
+      slug: "only-one",
+      path: "wiki/entities/only-one.md",
+      type: "entity",
+      title: "Only One",
+      tags: [],
+    },
+  ]
+
+  const result = await runDuplicateDetection(
+    "/Project",
+    { provider: "openai", model: "gpt-4", apiKey: "test" },
+    { summaries },
+  )
+
+  expect(result).toEqual({ groups: [], scannedPageCount: 1 })
+  expect(mockedStreamChat).not.toHaveBeenCalled()
+})
+
+test("runDuplicateDetection invokes onProgress for loading and detecting", async () => {
+  const stages: string[] = []
+  mockedListDirectory.mockResolvedValue(wikiTree("/Project", 2))
+  mockedReadFile.mockImplementation(async (path) => {
+    const match = path.match(/page-(\d+)\.md$/)
+    const idx = match?.[1] ?? "0"
+    return entityMarkdown(`Page ${idx}`)
+  })
+
+  await runDuplicateDetection(
+    "/Project",
+    { provider: "openai", model: "gpt-4", apiKey: "test" },
+    {
+      onProgress: (stage) => {
+        stages.push(stage)
+      },
+    },
+  )
+
+  expect(stages).toEqual(["loading", "detecting"])
+})

src/lib/dedup-runner.ts

@@ -21,6 +21,84 @@ import {
 } from "./dedup"
 import { loadNotDuplicates } from "./dedup-storage"
 
+const WIKI_READ_CONCURRENCY = 12
+const WIKI_WRITE_CONCURRENCY = 12
+
+export type DedupMergeStage = "loading" | "merging" | "writing"
+
+export type DedupScanStage = "loading" | "detecting"
+
+export interface ExecuteMergeOptions {
+  signal?: AbortSignal
+  onProgress?: (stage: DedupMergeStage) => void
+}
+
+export interface RunDuplicateDetectionOptions {
+  signal?: AbortSignal
+  summaries?: EntitySummary[]
+  onProgress?: (stage: DedupScanStage) => void
+}
+
+export interface DuplicateDetectionResult {
+  groups: DuplicateGroup[]
+  scannedPageCount: number
+}
+
+/**
+ * Run `fn` over `items` with a bounded worker pool. Items where `fn`
+ * returns null/undefined are omitted from the result.
+ */
+export async function mapWithConcurrency<T, R>(
+  items: readonly T[],
+  concurrency: number,
+  fn: (item: T) => Promise<R | null | undefined>,
+): Promise<R[]> {
+  if (items.length === 0) return []
+
+  const limit = Math.max(1, concurrency)
+  const results: R[] = []
+  let index = 0
+
+  async function worker(): Promise<void> {
+    while (true) {
+      const i = index++
+      if (i >= items.length) return
+      const result = await fn(items[i])
+      if (result !== null && result !== undefined) {
+        results.push(result)
+      }
+    }
+  }
+
+  await Promise.all(
+    Array.from({ length: Math.min(limit, items.length) }, () => worker()),
+  )
+  return results
+}
+
+async function runWithConcurrency<T>(
+  items: readonly T[],
+  concurrency: number,
+  fn: (item: T) => Promise<void>,
+): Promise<void> {
+  if (items.length === 0) return
+
+  const limit = Math.max(1, concurrency)
+  let index = 0
+
+  async function worker(): Promise<void> {
+    while (true) {
+      const i = index++
+      if (i >= items.length) return
+      await fn(items[i])
+    }
+  }
+
+  await Promise.all(
+    Array.from({ length: Math.min(limit, items.length) }, () => worker()),
+  )
+}
+
 /**
  * Wrap streamChat into the (system, user, signal) → string shape
  * the dedup module expects. Same pattern page-merge uses — keeps
@@ -91,20 +169,20 @@ export async function loadAllEntitySummaries(
 ): Promise<EntitySummary[]> {
   const pp = normalizePath(projectPath)
   const tree = await listDirectory(pp)
-  const out: EntitySummary[] = []
+  const nodes: FileNode[] = []
   for (const prefix of ["wiki/entities", "wiki/concepts"]) {
-    for (const node of walkMd(tree, prefix)) {
-      try {
-        const content = await readFile(node.path)
-        const rel = toWikiRelative(pp, node.path)
-        const summary = extractEntitySummary(rel, content)
-        if (summary) out.push(summary)
-      } catch {
-        // best-effort — skip unreadable pages
-      }
-    }
+    nodes.push(...walkMd(tree, prefix))
   }
-  return out
+
+  return mapWithConcurrency(nodes, WIKI_READ_CONCURRENCY, async (node) => {
+    try {
+      const content = await readFile(node.path)
+      const rel = toWikiRelative(pp, node.path)
+      return extractEntitySummary(rel, content)
+    } catch {
+      return null
+    }
+  })
 }
 
 /** Read every .md under wiki/ as { path, content }. The path is
@@ -114,16 +192,16 @@ export async function loadAllWikiPages(
 ): Promise<{ path: string; content: string }[]> {
   const pp = normalizePath(projectPath)
   const tree = await listDirectory(pp)
-  const out: { path: string; content: string }[] = []
-  for (const node of walkMd(tree, "wiki")) {
+  const nodes = [...walkMd(tree, "wiki")]
+
+  return mapWithConcurrency(nodes, WIKI_READ_CONCURRENCY, async (node) => {
     try {
       const content = await readFile(node.path)
-      out.push({ path: toWikiRelative(pp, node.path), content })
+      return { path: toWikiRelative(pp, node.path), content }
     } catch {
-      // ignore
+      return null
     }
-  }
-  return out
+  })
 }
 
 /**
@@ -134,16 +212,24 @@ export async function loadAllWikiPages(
 export async function runDuplicateDetection(
   projectPath: string,
   llmConfig: LlmConfig,
-  options: { signal?: AbortSignal } = {},
-): Promise<DuplicateGroup[]> {
-  const summaries = await loadAllEntitySummaries(projectPath)
-  if (summaries.length < 2) return []
+  options: RunDuplicateDetectionOptions = {},
+): Promise<DuplicateDetectionResult> {
+  options.onProgress?.("loading")
+  const summaries =
+    options.summaries ?? (await loadAllEntitySummaries(projectPath))
+
+  if (summaries.length < 2) {
+    return { groups: [], scannedPageCount: summaries.length }
+  }
+
+  options.onProgress?.("detecting")
   const notDup = await loadNotDuplicates(projectPath)
   const llm = buildDedupLlmCall(llmConfig)
-  return detectDuplicateGroups(summaries, llm, {
+  const groups = await detectDuplicateGroups(summaries, llm, {
     signal: options.signal,
     notDuplicates: notDup,
   })
+  return { groups, scannedPageCount: summaries.length }
 }
 
 /**
@@ -167,11 +253,13 @@ export async function executeMerge(
   group: DuplicateGroup,
   canonicalSlug: string,
   llmConfig: LlmConfig,
-  options: { signal?: AbortSignal } = {},
+  options: ExecuteMergeOptions = {},
 ): Promise<MergeResult> {
   const pp = normalizePath(projectPath)
+  const { signal, onProgress } = options
 
   // 1. Resolve each group slug to its actual on-disk path + content
+  onProgress?.("loading")
   const allPages = await loadAllWikiPages(pp)
   const pathBySlug = new Map<string, string>()
   for (const p of allPages) {
@@ -199,6 +287,7 @@ export async function executeMerge(
   const otherPages = allPages.filter((p) => !groupPaths.has(p.path))
 
   const llm = buildDedupLlmCall(llmConfig)
+  onProgress?.("merging")
   const result = await mergeDuplicateGroup(
     {
       group: groupPages,
@@ -206,36 +295,38 @@ export async function executeMerge(
       otherWikiPages: otherPages,
     },
     llm,
-    { signal: options.signal },
+    { signal },
   )
 
+  onProgress?.("writing")
+
   // 2. Snapshot backup before any writes. If a write fails partway
   //    through, the user has the pre-merge state intact in
   //    .qmai/page-history/.
   const stamp = new Date().toISOString().replace(/[:.]/g, "-")
   const backupDir = `${pp}/.qmai/page-history/dedup-${stamp}`
-  for (const b of result.backup) {
+  await runWithConcurrency(result.backup, WIKI_WRITE_CONCURRENCY, async (b) => {
     const sanitized = b.path.replace(/[/\\]/g, "_")
     await writeFile(`${backupDir}/${sanitized}`, b.content)
-  }
+  })
 
   // 3. Write canonical
   await writeFile(`${pp}/${result.canonicalPath}`, result.canonicalContent)
 
   // 4. Apply rewrites
-  for (const r of result.rewrites) {
+  await runWithConcurrency(result.rewrites, WIKI_WRITE_CONCURRENCY, async (r) => {
     await writeFile(`${pp}/${r.path}`, r.newContent)
-  }
+  })
 
   // 5. Delete merged-away pages
-  for (const dead of result.pagesToDelete) {
+  await runWithConcurrency(result.pagesToDelete, WIKI_WRITE_CONCURRENCY, async (dead) => {
     try {
       await deleteFile(`${pp}/${dead}`)
     } catch (err) {
       // Surface as a warning — backup is still safe.
       console.warn(`[dedup] failed to delete ${dead}: ${err}`)
     }
-  }
+  })
 
   // 6. Rewrite index.md to drop merged-away entries.
   const indexPath = `${pp}/wiki/index.md`