tui.snapshot.ts 37 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852
  1. import { cp, mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'
  2. import { tmpdir } from 'node:os'
  3. import { basename, dirname, isAbsolute, join, relative, sep } from 'node:path'
  4. import { fileURLToPath } from 'node:url'
  5. import { afterAll, describe, expect, it, vi } from 'vitest'
  6. import { Context } from 'cordis'
  7. import { scrubRequestHeaders, tokenizeSessionFixtureCwd } from '@deepseek-ai/dsh-acp-snapshot'
  8. import type { Agent } from '@deepseek-ai/dsh-agent'
  9. import * as AgentCore from '@deepseek-ai/dsh-agent-spine-demo'
  10. import { addHarnessSourceSection } from '@deepseek-ai/dsh-app-boot'
  11. import { LocalBashExecutor } from '@deepseek-ai/dsh-bash-local'
  12. import LocalSubprocessService from '@deepseek-ai/dsh-subprocess-local'
  13. import WorkerCodeRuntime from '@deepseek-ai/dsh-code-runtime-worker'
  14. import CommandService from '@deepseek-ai/dsh-commands'
  15. import * as CommandCompact from '@deepseek-ai/dsh-command-compact'
  16. import { BasicCompactService } from '@deepseek-ai/dsh-compact-basic'
  17. import type { SummarizationInput } from '@deepseek-ai/dsh-compact-basic/src/summarizer.ts'
  18. import LocalFileSystem from '@deepseek-ai/dsh-fs-local'
  19. import * as FsPolicy from '@deepseek-ai/dsh-fs-policy'
  20. import { createUserMessage } from '@deepseek-ai/dsh-llm'
  21. import * as ToolFs from '@deepseek-ai/dsh-tool-fs'
  22. import * as LlmDeepSeek from '@deepseek-ai/dsh-llm-deepseek'
  23. import { installLlmReplay, parseSessionLog } from '@deepseek-ai/dsh-llm-replay'
  24. import PlanModeService from '@deepseek-ai/dsh-plan-mode'
  25. import TokenMeterService from '@deepseek-ai/dsh-token-meter'
  26. import { packChunkRuns, SessionId, type Session, type SessionEvent } from '@deepseek-ai/dsh-session'
  27. import SubagentService from '@deepseek-ai/dsh-subagent'
  28. import * as SubagentSpawn from '@deepseek-ai/dsh-subagent-spawn'
  29. import * as ToolSubagent from '@deepseek-ai/dsh-tool-subagent'
  30. import * as ToolCordis from '@deepseek-ai/dsh-tool-cordis'
  31. import * as ToolTodo from '@deepseek-ai/dsh-tool-todo'
  32. import * as ToolRalph from '@deepseek-ai/dsh-tool-ralph'
  33. import * as ToolWorkflow from '@deepseek-ai/dsh-tool-workflow'
  34. import { createTuiChat, FILE_REFERENCE_PROMPT, TuiPromptService } from '@deepseek-ai/dsh-tui'
  35. import LocalSpillStore from '@deepseek-ai/dsh-spill-local'
  36. import * as SpillPolicy from '@deepseek-ai/dsh-spill-policy'
  37. import UserInteractionService from '@deepseek-ai/dsh-user-interaction'
  38. import WorkerWorkflowEngine from '@deepseek-ai/dsh-workflow-workerthread'
  39. import { HeadlessTerminal } from '../../../packages/ui/tui/tests/headless-terminal.ts'
  40. const SNAPSHOTS_DIR = join(dirname(fileURLToPath(import.meta.url)), 'snapshots')
  41. // Keep pre-normalization layout widths identical across macOS and Linux.
  42. const SNAPSHOT_TMP_ROOT = process.platform === 'win32' ? tmpdir() : '/tmp'
  43. const PROVIDERS = [{ id: 'deepseek-official', models: [{ id: 'deepseek-v4-flash', contextWindow: 128_000 }] }]
  44. const UUID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi
  45. type SnapshotMode = 'replay' | 'record' | 'refresh'
  46. type Composition = 'native' | 'code' | 'advanced'
  47. type ScenarioInteraction = 'skill-invocation-policy'
  48. interface Scenario {
  49. name: string
  50. /** Replay fixture owned by an earlier scenario, for a derived presentation case. */
  51. fixture?: string
  52. composition: Composition
  53. expectedTools: string[]
  54. expectedEventCounts?: Record<string, number>
  55. childSessions?: number
  56. enterPlanMode?: boolean
  57. leavePlanModeAfterFirstTurn?: boolean
  58. recorded: boolean
  59. seedWorkspace?: boolean
  60. /** Add the launcher's model-visible DSH source checkout at this fixed path. */
  61. harnessSourceRoot?: string
  62. /** Replace the real `pwd` result with a portable fixed-length workspace path. */
  63. normalizePwdResult?: boolean
  64. /**
  65. * Load the opt-in `todo_write` tool for this scenario. The shipped TUI
  66. * config omits it, so only the todo-plan scenario (the enabled-path proof)
  67. * mounts it; the rest cover the default, todo-free composition.
  68. */
  69. enableTodo?: boolean
  70. /**
  71. * Mount the spill stack (local backend + policy) with this inline cap, as the
  72. * shipped configs do. The dispatch-spill scenario proves the durable
  73. * `tool/code-dispatch` copy of an oversized sub-result is bounded to a
  74. * preview + locator while the program value stays whole.
  75. */
  76. spillMaxInlineBytes?: number
  77. /** Run scenario-specific terminal input instead of replaying recorded user prompts. */
  78. interaction?: ScenarioInteraction
  79. /**
  80. * Mount a deterministic compaction backend plus `/compact`, then run the
  81. * human command with a held summary while a prompt and injected context
  82. * arrive. Proves queued input waits for the standalone bracket's durability
  83. * checkpoint instead of racing the replacement.
  84. */
  85. manualCompact?: boolean
  86. }
  87. const SCENARIOS: Scenario[] = [
  88. {
  89. name: 'multi-turn-conversation',
  90. composition: 'native',
  91. expectedTools: [],
  92. expectedEventCounts: { 'plan/mode': 2 },
  93. enterPlanMode: true,
  94. leavePlanModeAfterFirstTurn: true,
  95. recorded: true,
  96. },
  97. {
  98. name: 'queued-manual-compact',
  99. fixture: 'multi-turn-conversation',
  100. composition: 'native',
  101. expectedTools: [],
  102. recorded: false,
  103. manualCompact: true,
  104. },
  105. {
  106. name: 'todo-plan',
  107. composition: 'native',
  108. expectedTools: ['todo_write'],
  109. expectedEventCounts: { 'todo/write': 1 },
  110. recorded: true,
  111. enableTodo: true,
  112. },
  113. {
  114. name: 'bash-terminal-card',
  115. composition: 'native',
  116. expectedTools: ['bash'],
  117. recorded: true,
  118. },
  119. {
  120. name: 'source-checkout-workdir',
  121. composition: 'native',
  122. expectedTools: ['bash'],
  123. recorded: true,
  124. harnessSourceRoot: '/opt/dsh-source',
  125. normalizePwdResult: true,
  126. },
  127. {
  128. name: 'parallel-file-reads',
  129. composition: 'native',
  130. expectedTools: ['read', 'read'],
  131. recorded: true,
  132. seedWorkspace: true,
  133. },
  134. {
  135. name: 'skill-invocation-policy',
  136. composition: 'native',
  137. expectedTools: [],
  138. recorded: false,
  139. seedWorkspace: true,
  140. interaction: 'skill-invocation-policy',
  141. },
  142. {
  143. name: 'code-mode',
  144. composition: 'code',
  145. expectedTools: ['run_code'],
  146. expectedEventCounts: { 'tool/code-dispatch': 2 },
  147. recorded: true,
  148. },
  149. {
  150. name: 'code-mode-dispatch-spill',
  151. composition: 'code',
  152. expectedTools: ['run_code'],
  153. expectedEventCounts: { 'tool/code-dispatch-start': 1, 'tool/code-dispatch': 1 },
  154. recorded: true,
  155. spillMaxInlineBytes: 600,
  156. },
  157. {
  158. name: 'dynamic-workflow',
  159. composition: 'native',
  160. expectedTools: ['workflow'],
  161. childSessions: 1,
  162. recorded: true,
  163. },
  164. {
  165. name: 'cordis-dynamic-toolchain',
  166. composition: 'advanced',
  167. expectedTools: ['cordis_mount', 'run_code', 'subagent', 'workflow', 'cordis_unmount'],
  168. expectedEventCounts: { 'tool/code-dispatch': 1 },
  169. childSessions: 2,
  170. recorded: false,
  171. },
  172. ]
  173. function snapshotModeFromEnv(value: string | undefined): SnapshotMode {
  174. if (value === undefined || value === '' || value === 'replay') return 'replay'
  175. if (value === 'record' || value === 'refresh') return value
  176. throw new Error(`DSH_SNAPSHOT must be replay, record, or refresh; got ${JSON.stringify(value)}`)
  177. }
  178. const MODE = snapshotModeFromEnv(process.env.DSH_SNAPSHOT)
  179. const observedScenarios = new Set<string>()
  180. const workerState = Reflect.get(globalThis, '__vitest_worker__') as
  181. | { readonly config?: { readonly testNamePattern?: RegExp } }
  182. | undefined
  183. // Worker argv omits the parent CLI's `-t`; the serialized runner config is the
  184. // authoritative distinction between a focused replay and the full suite.
  185. const TEST_NAME_FILTERED = workerState?.config?.testNamePattern !== undefined
  186. /**
  187. * Deterministic keyless summary that pauses so the scenario can submit a real
  188. * prompt and inject context while manual compaction holds turn admission.
  189. */
  190. class DeferredSnapshotCompactService extends BasicCompactService {
  191. readonly summaryStarted = Promise.withResolvers<undefined>()
  192. readonly releaseSummary = Promise.withResolvers<undefined>()
  193. override async summarize(
  194. _input: SummarizationInput,
  195. _agent: Agent,
  196. signal?: AbortSignal,
  197. ): Promise<{ summary: [{ type: 'text'; text: string }]; provider: string; model: string }> {
  198. this.summaryStarted.resolve(undefined)
  199. await this.releaseSummary.promise
  200. signal?.throwIfAborted()
  201. return {
  202. summary: [{ type: 'text', text: 'Keyless manual compaction checkpoint.' }],
  203. provider: 'snapshot',
  204. model: 'snapshot-compactor',
  205. }
  206. }
  207. }
  208. /** Seed between-turn model-visible history without inventing a loop execution. */
  209. function seedCompactableHistory(agent: Agent): void {
  210. agent.inject(createUserMessage({
  211. content: [{ type: 'text', text: 'Older snapshot context. '.repeat(60) }],
  212. source: { kind: 'plugin', plugin: 'snapshot-seed' },
  213. }))
  214. }
  215. function snapshotDisplayPath(displayPath: string, cwd: string, displayCwd: string): string {
  216. const rel = relative(cwd, displayPath)
  217. if (rel === '') return displayCwd
  218. if (isAbsolute(rel) || rel === '..' || rel.startsWith(`..${sep}`)) return displayPath
  219. return `${displayCwd}/${rel.split(sep).join('/')}`
  220. }
  221. function scenarioDir(scenario: Scenario): string {
  222. return join(SNAPSHOTS_DIR, scenario.name)
  223. }
  224. /** Directory owning the replay fixture: the scenario's own, or the one it derives from. */
  225. function fixtureDir(scenario: Scenario): string {
  226. return join(SNAPSHOTS_DIR, scenario.fixture ?? scenario.name)
  227. }
  228. function childFixturePaths(scenario: Scenario): string[] {
  229. return Array.from(
  230. { length: scenario.childSessions ?? 0 },
  231. (_, index) => join(fixtureDir(scenario), `session.${index + 1}.jsonl`),
  232. )
  233. }
  234. function userPrompts(rawLog: string): string[] {
  235. return parseSessionLog(rawLog).flatMap((event) => {
  236. if (event.type !== 'user/message' || event.data.source.kind !== 'user') return []
  237. const text = event.data.content
  238. .filter(block => block.type === 'text')
  239. .map(block => block.text)
  240. .join('')
  241. return text.length > 0 ? [text] : []
  242. })
  243. }
  244. function rawSessionLog(session: Session): string {
  245. return [
  246. JSON.stringify({ type: 'session', ...session.header }),
  247. ...packChunkRuns(session.events).map(record => JSON.stringify(record)),
  248. '',
  249. ].join('\n')
  250. }
  251. async function materializeFixtureCwd(fixtureFile: string, cwd: string, replayRoot: string): Promise<string> {
  252. const realized = join(replayRoot, basename(fixtureFile))
  253. await writeFile(realized, (await readFile(fixtureFile, 'utf8')).split('{{cwd}}').join(cwd))
  254. return realized
  255. }
  256. function normalizeTerminalSnapshot(snapshot: string, cwd: string, displayCwd: string): string {
  257. return snapshot
  258. .split(`/private${cwd}`).join('/workspace/project')
  259. .split(displayCwd).join('/workspace/project')
  260. .split(cwd).join('/workspace/project')
  261. .replace(UUID_RE, '{{uuid}}')
  262. }
  263. async function settleTerminal(terminal: HeadlessTerminal): Promise<void> {
  264. let stable = 0
  265. for (let attempt = 0; attempt < 20 && stable < 3; attempt++) {
  266. const before = terminal.frames
  267. await new Promise(resolve => setTimeout(resolve, 10))
  268. await terminal.flush()
  269. stable = terminal.frames === before ? stable + 1 : 0
  270. }
  271. if (stable < 3) throw new Error('TUI frames did not quiesce within 200ms')
  272. }
  273. /** Bound deterministic in-process coordination waits with actionable state. */
  274. async function snapshotDeadline<T>(
  275. operation: Promise<T>,
  276. detail: () => string,
  277. ): Promise<T> {
  278. let timer: ReturnType<typeof setTimeout> | undefined
  279. try {
  280. return await Promise.race([
  281. operation,
  282. new Promise<never>((_resolve, reject) => {
  283. timer = setTimeout(() => { reject(new Error(detail())) }, 5_000)
  284. }),
  285. ])
  286. } finally {
  287. if (timer !== undefined) clearTimeout(timer)
  288. }
  289. }
  290. async function mountScenarioContext(
  291. scenario: Scenario,
  292. cwd: string,
  293. displayCwd: string,
  294. fixtureFile: string,
  295. childFiles: string[],
  296. replayRoot: string | undefined,
  297. ): Promise<Context> {
  298. class SnapshotLocalFileSystem extends LocalFileSystem {
  299. override async resolve(
  300. path: string,
  301. opts?: { cwd?: string; signal?: AbortSignal },
  302. ): Promise<Awaited<ReturnType<LocalFileSystem['resolve']>>> {
  303. const target = await super.resolve(path, opts)
  304. return { ...target, displayPath: snapshotDisplayPath(target.displayPath, cwd, displayCwd) }
  305. }
  306. }
  307. const ctx = new Context()
  308. await ctx.plugin(AgentCore, {
  309. agents: [],
  310. dshHome: join(cwd, '.dsh'),
  311. workspaceContext: false,
  312. tools: { mode: scenario.composition === 'code' ? 'code' : scenario.composition === 'advanced' ? 'both' : 'native' },
  313. skills: { local: { agentsHome: join(cwd, '.agents') } },
  314. })
  315. if (scenario.harnessSourceRoot !== undefined) addHarnessSourceSection(ctx, scenario.harnessSourceRoot)
  316. await ctx.plugin(TokenMeterService)
  317. if (scenario.manualCompact === true) {
  318. await ctx.plugin(DeferredSnapshotCompactService, { auto: false })
  319. }
  320. await ctx.plugin(LocalSubprocessService)
  321. await ctx.plugin(LocalBashExecutor, { cwd, timeoutMs: 30_000 })
  322. await ctx.plugin(SnapshotLocalFileSystem, { cwd: '/' })
  323. await ctx.plugin(FsPolicy)
  324. await ctx.plugin(ToolFs)
  325. await ctx.plugin(UserInteractionService)
  326. await ctx.plugin(TuiPromptService)
  327. // todo_write is opt-in: only the todo-plan scenario mounts it, matching the shipped
  328. // config that omits it. The other scenarios prove the default todo-free composition.
  329. if (scenario.enableTodo === true) await ctx.plugin(ToolTodo)
  330. await ctx.plugin(SubagentService)
  331. await ctx.plugin(SubagentSpawn, { providerName: 'spawn' })
  332. await ctx.plugin(ToolSubagent, { provider: 'spawn', toolName: 'subagent', enableRunInBackground: false })
  333. await ctx.plugin(WorkerWorkflowEngine, { provider: 'spawn' })
  334. await ctx.plugin(ToolWorkflow)
  335. await ctx.plugin(ToolRalph)
  336. await ctx.plugin(CommandService)
  337. if (scenario.manualCompact === true) await ctx.plugin(CommandCompact)
  338. if (scenario.enterPlanMode === true) {
  339. await ctx.plugin(PlanModeService, { section: 'Snapshot plan mode instructions.' })
  340. }
  341. if (scenario.composition === 'code' || scenario.composition === 'advanced') {
  342. await ctx.plugin(WorkerCodeRuntime, {})
  343. }
  344. if (scenario.spillMaxInlineBytes !== undefined) {
  345. await ctx.plugin(LocalSpillStore, { root: join(cwd, '.spill') })
  346. await ctx.plugin(SpillPolicy, { maxInlineBytes: scenario.spillMaxInlineBytes })
  347. }
  348. if (scenario.composition === 'advanced') await ctx.plugin(ToolCordis, { vmTimeoutMs: 5_000 })
  349. if (MODE === 'record' && scenario.recorded) {
  350. await ctx.plugin(LlmDeepSeek)
  351. } else {
  352. if (replayRoot === undefined) throw new Error('replay mode requires an isolated fixture directory')
  353. // Recorded model text may name the generated cwd. Realize the portable token
  354. // outside that cwd so tools see only the scenario workspace during replay.
  355. const replayFile = await materializeFixtureCwd(fixtureFile, cwd, replayRoot)
  356. const replayChildFiles = await Promise.all(childFiles.map(file => materializeFixtureCwd(file, cwd, replayRoot)))
  357. installLlmReplay(ctx, { file: replayFile, childFiles: replayChildFiles, providers: PROVIDERS })
  358. }
  359. return ctx
  360. }
  361. interface ScenarioResult {
  362. terminal: string
  363. parent: Session
  364. children: Session[]
  365. workflowEvents: string[]
  366. }
  367. async function runScenario(scenario: Scenario): Promise<ScenarioResult> {
  368. const snapshotTime = new Date(2026, 6, 21, 12, 0, 0).getTime()
  369. const clock = vi.spyOn(Date, 'now').mockReturnValue(snapshotTime)
  370. const fixtureFile = join(fixtureDir(scenario), 'session.jsonl')
  371. const childFiles = childFixturePaths(scenario)
  372. const prompts = userPrompts(await readFile(fixtureFile, 'utf8'))
  373. if (scenario.interaction === undefined) {
  374. expect(prompts.length, `${scenario.name} must carry at least one recorded user prompt`).toBeGreaterThan(0)
  375. }
  376. const cwd = await mkdtemp(join(SNAPSHOT_TMP_ROOT, `dsh-tui-snapshot-${scenario.name}-`))
  377. const displayCwd = `/tmp/${basename(cwd)}`
  378. let replayRoot: string | undefined
  379. let ctx: Context | undefined
  380. let controller: ReturnType<typeof createTuiChat> | undefined
  381. const terminal = new HeadlessTerminal(100, 36)
  382. try {
  383. if (!(MODE === 'record' && scenario.recorded)) {
  384. replayRoot = await mkdtemp(join(SNAPSHOT_TMP_ROOT, `dsh-tui-replay-${scenario.name}-`))
  385. }
  386. if (scenario.seedWorkspace === true) {
  387. const source = join(fixtureDir(scenario), 'workspace')
  388. await cp(source, cwd, { recursive: true })
  389. }
  390. ctx = await mountScenarioContext(scenario, cwd, displayCwd, fixtureFile, childFiles, replayRoot)
  391. if (scenario.normalizePwdResult === true) {
  392. ctx.on('tools/post-execute', async (exec, result, next) => {
  393. const args = exec.arguments as { command?: unknown }
  394. return exec.name === 'bash' && args.command === 'pwd' && !result.isError
  395. ? { kind: 'accept', content: [{ type: 'text', text: '/workspace/project\n' }] }
  396. : next()
  397. })
  398. }
  399. const disposedSessions: Session[] = []
  400. ctx.on('session/disposed', (session) => { disposedSessions.push(session) })
  401. const workflowEvents: string[] = []
  402. for (const name of ['workflow/start', 'workflow/phase', 'workflow/agent-start', 'workflow/agent-end', 'workflow/end'] as const) {
  403. ctx.on(name, () => { workflowEvents.push(name) })
  404. }
  405. const handle = await ctx.agents.create({
  406. sessionId: SessionId('main-session'),
  407. meta: { cwd },
  408. agentOptions: { provider: 'deepseek-official', model: 'deepseek-v4-flash' },
  409. })
  410. const agent: Agent = handle.agent
  411. if (scenario.manualCompact === true) seedCompactableHistory(agent)
  412. controller = createTuiChat(ctx, {
  413. sessionId: 'main-session',
  414. theme: { color: true },
  415. showReasoning: true,
  416. title: 'DSH TUI snapshot',
  417. welcome: `Recorded replay: ${scenario.name}`,
  418. maxToolOutputLines: 8,
  419. }, {
  420. terminal,
  421. exit: () => {},
  422. formatCwd: () => displayCwd,
  423. })
  424. await settleTerminal(terminal)
  425. let interactionSnapshot: string | undefined
  426. if (scenario.interaction === 'skill-invocation-policy') {
  427. terminal.send('/skill')
  428. await settleTerminal(terminal)
  429. const discovery = normalizeTerminalSnapshot(
  430. await terminal.snapshot({ includeScrollback: true }),
  431. cwd,
  432. displayCwd,
  433. )
  434. expect(discovery).toContain('user-only-skill')
  435. expect(discovery).not.toContain('model-only-skill')
  436. terminal.send('\x03')
  437. await settleTerminal(terminal)
  438. const skillContext = ctx
  439. const skillTurnEnded = new Promise<void>((resolve) => {
  440. const detach = skillContext.on('session/event', (session, event) => {
  441. if (session !== agent.session || event.type !== 'turn/end') return
  442. detach()
  443. resolve()
  444. })
  445. })
  446. terminal.send('/skill:user-only-skill')
  447. terminal.send('\r')
  448. await skillTurnEnded
  449. await agent.whenIdle()
  450. await settleTerminal(terminal)
  451. const loaded = normalizeTerminalSnapshot(
  452. await terminal.snapshot({ includeScrollback: true }),
  453. cwd,
  454. displayCwd,
  455. )
  456. expect(loaded).toContain('USER-ONLY SKILL LOADED')
  457. terminal.send('/skill:model-only-skill')
  458. terminal.send('\r')
  459. await settleTerminal(terminal)
  460. const denied = normalizeTerminalSnapshot(
  461. await terminal.snapshot({ includeScrollback: true }),
  462. cwd,
  463. displayCwd,
  464. )
  465. expect(denied).toContain('model-only-skill')
  466. expect(denied).toContain('not available for user invocation.')
  467. expect(denied).not.toContain('MODEL-ONLY BODY MUST NOT LOAD')
  468. interactionSnapshot = [
  469. '=== skill autocomplete ===',
  470. discovery,
  471. '',
  472. '=== loaded exact invocation ===',
  473. loaded,
  474. '',
  475. '=== denied exact invocation ===',
  476. denied,
  477. ].join('\n')
  478. }
  479. let remainingPrompts = prompts
  480. let queuedPrompt: string | undefined
  481. let manualOrder: string[] | undefined
  482. let manualCommandId: string | undefined
  483. if (scenario.manualCompact === true) {
  484. expect(prompts.length, 'queued manual compaction needs a second replayed prompt').toBeGreaterThanOrEqual(2)
  485. queuedPrompt = prompts.at(-1)
  486. remainingPrompts = prompts.slice(0, -1)
  487. }
  488. if (scenario.enterPlanMode === true) {
  489. const firstPrompt = prompts[0]!
  490. terminal.send(`/plan ${firstPrompt}`)
  491. terminal.send('\r')
  492. await agent.whenIdle()
  493. await settleTerminal(terminal)
  494. remainingPrompts = prompts.slice(1)
  495. }
  496. if (scenario.leavePlanModeAfterFirstTurn === true) {
  497. terminal.send('/plan off')
  498. terminal.send('\r')
  499. await settleTerminal(terminal)
  500. }
  501. for (const prompt of remainingPrompts) {
  502. const admitted = agent.session.events.filter(event =>
  503. event.type === 'user/message' && event.data.source.kind === 'user').length
  504. terminal.send(prompt)
  505. terminal.send('\r')
  506. await terminal.flush()
  507. await expect.poll(() => agent.session.events.filter(event =>
  508. event.type === 'user/message' && event.data.source.kind === 'user').length).toBe(admitted + 1)
  509. await agent.whenIdle()
  510. await settleTerminal(terminal)
  511. }
  512. if (scenario.manualCompact === true && queuedPrompt !== undefined) {
  513. terminal.send('/help')
  514. terminal.send('\r')
  515. await settleTerminal(terminal)
  516. expect(await terminal.snapshot({ includeScrollback: true }))
  517. .toContain('/compact — Compact older conversation history')
  518. const compact = ctx.compact as DeferredSnapshotCompactService
  519. const inbox: string[] = []
  520. manualOrder = []
  521. ctx.on('agent/inbox/enqueue', (subject, item) => {
  522. if (subject === agent) inbox.push(`enqueue:${item.placement}:${item.id}`)
  523. })
  524. ctx.on('agent/inbox/dequeue', (subject, message) => {
  525. if (subject === agent) inbox.push(`dequeue:${message.id}`)
  526. })
  527. ctx.on('session/event', (session, event) => {
  528. if (session !== agent.session) return
  529. if (event.type === 'command/run' && event.data.name === 'compact') {
  530. manualCommandId = event.data.commandId
  531. manualOrder?.push('command/run')
  532. }
  533. if (event.type === 'command/done' && event.data.commandId === manualCommandId) {
  534. manualOrder?.push('command/done')
  535. }
  536. if (event.type.startsWith('compact/')) manualOrder?.push(event.type)
  537. if (event.type === 'user/message'
  538. && event.data.source.kind === 'plugin'
  539. && event.data.source.plugin === 'compact') manualOrder?.push('checkpoint')
  540. if (event.type === 'turn/start') manualOrder?.push(`turn/start:${event.data.trigger.kind}`)
  541. })
  542. ctx.on('session/flush', (session) => {
  543. if (session === agent.session) manualOrder?.push('flush')
  544. })
  545. terminal.send('/compact')
  546. terminal.send('\r')
  547. await terminal.flush()
  548. await snapshotDeadline(compact.summaryStarted.promise, () =>
  549. `manual summary did not start; status=${agent.status}; tail=${
  550. agent.session.events.slice(-8).map(event => event.type).join(',')
  551. }`)
  552. clock.mockReturnValue(snapshotTime + 1_000)
  553. await settleTerminal(terminal)
  554. await expect.poll(() => terminal.snapshot()).toContain('dsh ⊙')
  555. await expect.poll(() => terminal.snapshot()).toContain('Context being compacted 1.0s')
  556. const liveCompaction = await terminal.snapshot()
  557. expect(liveCompaction.indexOf('Context being compacted 1.0s')).toBeLessThan(liveCompaction.indexOf('dsh ⊙'))
  558. clock.mockReturnValue(snapshotTime)
  559. // Real keystrokes: the prompt keeps its ordinary queue identity while
  560. // admission is reserved, and an injection appends immediately.
  561. terminal.send(queuedPrompt)
  562. terminal.send('\r')
  563. await terminal.flush()
  564. await expect.poll(() => inbox.length).toBe(1)
  565. agent.inject(createUserMessage({
  566. content: [{ type: 'text', text: 'Injected while compaction was running.' }],
  567. source: { kind: 'plugin', plugin: 'snapshot-injector' },
  568. }))
  569. expect(inbox[0]).toMatch(/^enqueue:queued:/u)
  570. expect(agent.status).toBe('idle')
  571. expect(agent.session.events.some(event => event.type === 'user/message'
  572. && event.data.source.kind === 'user'
  573. && event.data.content.some(block => block.type === 'text' && block.text === queuedPrompt))).toBe(false)
  574. const idle = agent.whenIdle()
  575. compact.releaseSummary.resolve(undefined)
  576. await snapshotDeadline(idle, () =>
  577. `manual compaction did not reach idle; status=${agent.status}; order=${manualOrder?.join(',') ?? ''}; tail=${
  578. agent.session.events.slice(-12).map(event => event.type).join(',')
  579. }`)
  580. await settleTerminal(terminal)
  581. expect(inbox).toEqual([inbox[0], `dequeue:${inbox[0]?.slice('enqueue:queued:'.length) ?? ''}`])
  582. }
  583. const events: SessionEvent[] = [...agent.session.events]
  584. const firstHeader = events.find(event => event.type === 'request/header')
  585. expect(firstHeader?.type === 'request/header' && firstHeader.data.header.system)
  586. .toContain(FILE_REFERENCE_PROMPT)
  587. if (scenario.harnessSourceRoot !== undefined) {
  588. expect(firstHeader?.type === 'request/header' && firstHeader.data.header.system)
  589. .toContain(`The DeepSeek Harness implementation checkout is at ${scenario.harnessSourceRoot}. The checkout location and current working directory are separate values and may differ; never infer the working directory from this path. Use pwd to determine the current working directory. Use this checkout only to inspect or extend DSH itself.`)
  590. }
  591. expect(events.filter(event => event.type === 'tool/call').map(event => event.data.name)).toEqual(scenario.expectedTools)
  592. for (const [type, count] of Object.entries(scenario.expectedEventCounts ?? {})) {
  593. expect(events.filter(event => event.type === type), `${scenario.name} must emit ${type}`).toHaveLength(count)
  594. }
  595. if (scenario.enterPlanMode === true) {
  596. expect(ctx.planMode.get(agent)).toEqual({
  597. active: scenario.leavePlanModeAfterFirstTurn !== true,
  598. })
  599. const planMode = events.find(event => event.type === 'plan/mode')
  600. if (planMode === undefined || firstHeader === undefined) {
  601. throw new Error('plan-mode command snapshot needs plan/mode before its first request/header')
  602. }
  603. expect(planMode.seq).toBeLessThan(firstHeader.seq)
  604. expect(firstHeader.data.header.system).toContain('Snapshot plan mode instructions.')
  605. const firstMessage = events.find(event => event.type === 'user/message')
  606. expect(firstMessage?.data.content).toEqual([{ type: 'text', text: prompts[0] }])
  607. }
  608. if (scenario.leavePlanModeAfterFirstTurn === true) {
  609. const planModes = events.filter(event => event.type === 'plan/mode')
  610. expect(planModes.map(event => event.data.active)).toEqual([true, false])
  611. const headers = events.filter(event => event.type === 'request/header')
  612. const exit = planModes[1]
  613. const afterExit = headers[1]
  614. if (exit === undefined || afterExit === undefined) {
  615. throw new Error('active plan exit snapshot needs a committed exit and changed request header')
  616. }
  617. expect(exit.seq).toBeLessThan(afterExit.seq)
  618. expect(afterExit.data.header.system).not.toContain('Snapshot plan mode instructions.')
  619. expect(events.filter(event => event.type === 'user/message' && event.data.source.kind === 'plugin').map(event => (event.data as { content: unknown }).content))
  620. .toContainEqual([{ type: 'text', text: 'The user switched this session back to the default mode.' }])
  621. }
  622. if (scenario.manualCompact === true) {
  623. const compactStart = events.find(event => event.type === 'compact/start')
  624. const compactSummary = events.find(event => event.type === 'compact/summary')
  625. const compactCheckpoint = events.find(event => event.type === 'user/message'
  626. && event.data.source.kind === 'plugin' && event.data.source.plugin === 'compact')
  627. const injectedEvent = events.find(event => event.type === 'user/message'
  628. && event.data.source.kind === 'plugin' && event.data.source.plugin === 'snapshot-injector')
  629. const compactEnd = events.find(event => event.type === 'compact/end')
  630. expect(compactStart?.data.turn).toBeNull()
  631. expect(compactEnd?.data.turn).toBeNull()
  632. expect(events.filter(event => event.type === 'compact/summary')).toHaveLength(1)
  633. if (compactStart === undefined || compactSummary === undefined
  634. || compactCheckpoint === undefined || injectedEvent === undefined
  635. || compactEnd === undefined) {
  636. throw new Error('manual compaction snapshot is missing its durable marker, summary, checkpoint, or injection')
  637. }
  638. // The markers are time points, not an exclusive container: unrelated
  639. // idle injection is allowed between them while the selected span stays stable.
  640. expect(compactStart.seq).toBeLessThan(injectedEvent.seq)
  641. expect(injectedEvent.seq).toBeLessThan(compactSummary.seq)
  642. expect(compactSummary.seq).toBeLessThan(compactCheckpoint.seq)
  643. expect(compactCheckpoint.seq).toBeLessThan(compactEnd.seq)
  644. const manualTimeline = manualOrder ?? []
  645. const commandRunIndex = manualTimeline.indexOf('command/run')
  646. const compactStartIndex = manualTimeline.indexOf('compact/start')
  647. const compactEndIndex = manualTimeline.indexOf('compact/end')
  648. const firstFlushIndex = manualTimeline.indexOf('flush')
  649. const queuedTurnIndex = manualTimeline.indexOf('turn/start:message')
  650. const commandDoneIndex = manualTimeline.indexOf('command/done')
  651. expect(manualTimeline.filter(item => item === 'command/run')).toHaveLength(1)
  652. expect(manualTimeline.filter(item => item === 'command/done')).toHaveLength(1)
  653. expect(compactStartIndex).toBeGreaterThan(commandRunIndex)
  654. expect(compactEndIndex).toBeGreaterThan(compactStartIndex)
  655. expect(firstFlushIndex).toBeGreaterThan(compactEndIndex)
  656. expect(queuedTurnIndex).toBeGreaterThan(firstFlushIndex)
  657. expect(commandDoneIndex).toBeGreaterThan(firstFlushIndex)
  658. const commandRun = events.find(event => event.type === 'command/run'
  659. && event.data.name === 'compact')
  660. const commandRunId = commandRun?.type === 'command/run'
  661. ? commandRun.data.commandId
  662. : undefined
  663. const commandDone = events.find(event => event.type === 'command/done'
  664. && event.data.commandId === commandRunId)
  665. expect(commandRun?.type === 'command/run' && commandRun.data).toEqual({
  666. commandId: commandRunId,
  667. name: 'compact',
  668. args: '',
  669. source: { kind: 'user' },
  670. })
  671. expect(commandDone?.type === 'command/done' && commandDone.data).toEqual({
  672. commandId: commandRunId,
  673. kind: 'success',
  674. text: 'Compacted 2 history items (~387 tokens).',
  675. })
  676. expect(commandRun !== undefined && commandRun.seq < compactStart.seq).toBe(true)
  677. expect(commandDone !== undefined && commandDone.seq > compactEnd.seq).toBe(true)
  678. expect(agent.session.surface.nodes).not.toContain(commandRun?.seq)
  679. expect(agent.session.surface.nodes).not.toContain(commandDone?.seq)
  680. // The command line itself never becomes a prompt.
  681. expect(events.some(event => event.type === 'user/message'
  682. && event.data.source.kind === 'user'
  683. && event.data.content.some(block => block.type === 'text' && block.text.trim() === '/compact'))).toBe(false)
  684. const derived = agent.session.deriveMessages().map(message => message.content
  685. .map(block => block.type === 'text' ? block.text : '')
  686. .join(''))
  687. const checkpoint = derived.findIndex(text => text.includes('Keyless manual compaction checkpoint.'))
  688. const injected = derived.findIndex(text => text.includes('Injected while compaction was running.'))
  689. const queued = derived.findIndex(text => text === queuedPrompt)
  690. expect(checkpoint).toBe(0)
  691. expect(injected).toBeGreaterThan(checkpoint)
  692. expect(queued).toBeGreaterThan(injected)
  693. expect(derived).not.toContain('/compact')
  694. expect(derived).not.toContain('Compacted 2 history items (~387 tokens).')
  695. expect(derived.filter(text => text.includes('Injected while compaction was running.'))).toHaveLength(1)
  696. expect(compactSummary.data.shadowedSeqs).not.toContain(injectedEvent.seq)
  697. const queuedTurn = events.findLast(event => event.type === 'turn/start')
  698. expect(queuedTurn !== undefined && compactEnd.seq < queuedTurn.seq).toBe(true)
  699. }
  700. if (scenario.spillMaxInlineBytes !== undefined) {
  701. // The REAL pipeline ran (tools execute on replay too): the durable
  702. // dispatch copy is bounded to a preview + locator under the run cwd,
  703. // while the outer result still carries the program's whole value.
  704. const dispatch = events.find(event => (event.type as string) === 'tool/code-dispatch')
  705. const content = (dispatch?.data as { content: { type: string; text?: string }[] }).content
  706. const text = content.filter(block => block.type === 'text').map(block => block.text ?? '').join('')
  707. expect(Buffer.byteLength(text, 'utf8')).toBeLessThanOrEqual(scenario.spillMaxInlineBytes)
  708. expect(text).toContain('Full formatted result stored at:')
  709. expect(text).toContain('.spill')
  710. }
  711. expect(events.filter(event => event.type === 'tool/result').every(event => !event.data.message.content[0].isError)).toBe(true)
  712. expect(events.filter(event => event.type === 'turn/end').every(event => event.data.reason.kind !== 'error')).toBe(true)
  713. if (scenario.name === 'dynamic-workflow' || scenario.name === 'cordis-dynamic-toolchain') {
  714. expect(workflowEvents).toEqual([
  715. 'workflow/start',
  716. 'workflow/phase',
  717. 'workflow/agent-start',
  718. 'workflow/agent-end',
  719. 'workflow/end',
  720. ])
  721. }
  722. expect(terminal.themeViolations(), `${scenario.name} must remain theme-agnostic`).toEqual([])
  723. const snapshot = interactionSnapshot ?? normalizeTerminalSnapshot(
  724. await terminal.snapshot({ includeScrollback: true }),
  725. cwd,
  726. displayCwd,
  727. )
  728. await handle.dispose()
  729. const children = disposedSessions
  730. .filter(session => session !== agent.session)
  731. .sort((a, b) => a.header.createdAt - b.header.createdAt)
  732. expect(children).toHaveLength(scenario.childSessions ?? 0)
  733. return { terminal: snapshot, parent: agent.session, children, workflowEvents }
  734. } finally {
  735. await controller?.dispose()
  736. await ctx?.fiber.dispose()
  737. await terminal.dispose()
  738. await rm(cwd, { recursive: true, force: true })
  739. if (replayRoot !== undefined) await rm(replayRoot, { recursive: true, force: true })
  740. clock.mockRestore()
  741. }
  742. }
  743. async function writeRecording(scenario: Scenario, result: ScenarioResult): Promise<void> {
  744. const dir = scenarioDir(scenario)
  745. await mkdir(dir, { recursive: true })
  746. await writeFile(
  747. join(dir, 'session.jsonl'),
  748. scrubRequestHeaders(tokenizeSessionFixtureCwd(rawSessionLog(result.parent))),
  749. )
  750. expect(result.children).toHaveLength(scenario.childSessions ?? 0)
  751. for (const [index, child] of result.children.entries()) {
  752. await writeFile(
  753. join(dir, `session.${index + 1}.jsonl`),
  754. scrubRequestHeaders(tokenizeSessionFixtureCwd(rawSessionLog(child))),
  755. )
  756. }
  757. }
  758. describe('TUI recorded-session terminal snapshots', () => {
  759. for (const scenario of SCENARIOS) {
  760. it(scenario.name, async () => {
  761. observedScenarios.add(scenario.name)
  762. const result = await runScenario(scenario)
  763. const terminalFile = join(scenarioDir(scenario), 'terminal.expected.txt')
  764. if (MODE === 'record' || MODE === 'refresh') {
  765. await mkdir(scenarioDir(scenario), { recursive: true })
  766. await writeFile(terminalFile, result.terminal)
  767. }
  768. if (MODE === 'record' && scenario.recorded) await writeRecording(scenario, result)
  769. await expect(result.terminal).toMatchFileSnapshot(terminalFile)
  770. }, 120_000)
  771. }
  772. })
  773. afterAll(async () => {
  774. const scenarioNames = SCENARIOS.map(scenario => scenario.name).sort()
  775. const observedNames = [...observedScenarios].sort()
  776. if (TEST_NAME_FILTERED) {
  777. expect(observedNames).not.toHaveLength(0)
  778. expect(scenarioNames).toEqual(expect.arrayContaining(observedNames))
  779. } else {
  780. expect(observedNames).toEqual(scenarioNames)
  781. }
  782. for (const [index, scenario] of SCENARIOS.entries()) {
  783. if (scenario.fixture === undefined) continue
  784. const sourceIndex = SCENARIOS.findIndex(candidate => candidate.name === scenario.fixture)
  785. expect(sourceIndex, `${scenario.name} fixture source ${scenario.fixture} must exist`).toBeGreaterThanOrEqual(0)
  786. expect(sourceIndex, `${scenario.name} fixture source must precede it`).toBeLessThan(index)
  787. const source = SCENARIOS[sourceIndex]
  788. expect(source?.fixture, `${scenario.name} fixture source must own its replay files`).toBeUndefined()
  789. expect(source?.recorded, `${scenario.name} fixture source must be recordable`).toBe(true)
  790. }
  791. const directories = (await readdir(SNAPSHOTS_DIR, { withFileTypes: true }))
  792. .filter(entry => entry.isDirectory())
  793. .map(entry => entry.name)
  794. .sort()
  795. expect(directories).toEqual(SCENARIOS.map(scenario => scenario.name).sort())
  796. for (const scenario of SCENARIOS) {
  797. const expected = [
  798. ...scenario.fixture === undefined ? ['session.jsonl'] : [],
  799. 'terminal.expected.txt',
  800. ...scenario.seedWorkspace === true && scenario.fixture === undefined ? ['workspace'] : [],
  801. ...Array.from({ length: scenario.childSessions ?? 0 }, (_, index) => `session.${index + 1}.jsonl`),
  802. ].sort()
  803. expect((await readdir(scenarioDir(scenario))).sort()).toEqual(expected)
  804. for (const fixture of ['session.jsonl', ...childFixturePaths(scenario).map(path => basename(path))]) {
  805. const content = await readFile(join(fixtureDir(scenario), fixture), 'utf8')
  806. expect(scrubRequestHeaders(content), `${scenario.name}/${fixture} carries request-header bulk`).toBe(content)
  807. }
  808. }
  809. })