tui.snapshot.ts 35 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809
  1. import { cp, mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'
  2. import { tmpdir } from 'node:os'
  3. import { basename, dirname, isAbsolute, join, relative, sep } from 'node:path'
  4. import { fileURLToPath } from 'node:url'
  5. import { afterAll, describe, expect, it, vi } from 'vitest'
  6. import { Context } from 'cordis'
  7. import { scrubRequestHeaders, tokenizeSessionFixtureCwd } from '@deepseek-ai/dsh-acp-snapshot'
  8. import type { Agent } from '@deepseek-ai/dsh-agent'
  9. import * as AgentCore from '@deepseek-ai/dsh-agent-spine-demo'
  10. import { LocalBashExecutor } from '@deepseek-ai/dsh-bash-local'
  11. import LocalSubprocessService from '@deepseek-ai/dsh-subprocess-local'
  12. import WorkerCodeRuntime from '@deepseek-ai/dsh-code-runtime-worker'
  13. import CommandService from '@deepseek-ai/dsh-commands'
  14. import * as CommandCompact from '@deepseek-ai/dsh-command-compact'
  15. import { BasicCompactService } from '@deepseek-ai/dsh-compact-basic'
  16. import type { SummarizationInput } from '@deepseek-ai/dsh-compact-basic/src/summarizer.ts'
  17. import LocalFileSystem from '@deepseek-ai/dsh-fs-local'
  18. import * as FsPolicy from '@deepseek-ai/dsh-fs-policy'
  19. import { createUserMessage } from '@deepseek-ai/dsh-llm'
  20. import * as ToolFs from '@deepseek-ai/dsh-tool-fs'
  21. import * as LlmDeepSeek from '@deepseek-ai/dsh-llm-deepseek'
  22. import { installLlmReplay, parseSessionLog } from '@deepseek-ai/dsh-llm-replay'
  23. import PlanModeService from '@deepseek-ai/dsh-plan-mode'
  24. import TokenMeterService from '@deepseek-ai/dsh-token-meter'
  25. import { packChunkRuns, SessionId, type Session, type SessionEvent } from '@deepseek-ai/dsh-session'
  26. import SubagentService from '@deepseek-ai/dsh-subagent'
  27. import * as SubagentSpawn from '@deepseek-ai/dsh-subagent-spawn'
  28. import * as ToolSubagent from '@deepseek-ai/dsh-tool-subagent'
  29. import * as ToolCordis from '@deepseek-ai/dsh-tool-cordis'
  30. import * as ToolTodo from '@deepseek-ai/dsh-tool-todo'
  31. import * as ToolRalph from '@deepseek-ai/dsh-tool-ralph'
  32. import * as ToolWorkflow from '@deepseek-ai/dsh-tool-workflow'
  33. import { createTuiChat, FILE_REFERENCE_PROMPT, TuiPromptService } from '@deepseek-ai/dsh-tui'
  34. import LocalSpillStore from '@deepseek-ai/dsh-spill-local'
  35. import * as SpillPolicy from '@deepseek-ai/dsh-spill-policy'
  36. import UserInteractionService from '@deepseek-ai/dsh-user-interaction'
  37. import WorkerWorkflowEngine from '@deepseek-ai/dsh-workflow-workerthread'
  38. import { HeadlessTerminal } from '../../../packages/ui/tui/tests/headless-terminal.ts'
  39. const SNAPSHOTS_DIR = join(dirname(fileURLToPath(import.meta.url)), 'snapshots')
  40. // Keep pre-normalization layout widths identical across macOS and Linux.
  41. const SNAPSHOT_TMP_ROOT = process.platform === 'win32' ? tmpdir() : '/tmp'
  42. const PROVIDERS = [{ id: 'deepseek-official', models: [{ id: 'deepseek-v4-flash', contextWindow: 128_000 }] }]
  43. const UUID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi
  44. type SnapshotMode = 'replay' | 'record' | 'refresh'
  45. type Composition = 'native' | 'code' | 'advanced'
  46. type ScenarioInteraction = 'skill-invocation-policy'
  47. interface Scenario {
  48. name: string
  49. /** Replay fixture owned by an earlier scenario, for a derived presentation case. */
  50. fixture?: string
  51. composition: Composition
  52. expectedTools: string[]
  53. expectedEventCounts?: Record<string, number>
  54. childSessions?: number
  55. enterPlanMode?: boolean
  56. leavePlanModeAfterFirstTurn?: boolean
  57. recorded: boolean
  58. seedWorkspace?: boolean
  59. /**
  60. * Load the opt-in `todo_write` tool for this scenario. The shipped TUI
  61. * config omits it, so only the todo-plan scenario (the enabled-path proof)
  62. * mounts it; the rest cover the default, todo-free composition.
  63. */
  64. enableTodo?: boolean
  65. /**
  66. * Mount the spill stack (local backend + policy) with this inline cap, as the
  67. * shipped configs do. The dispatch-spill scenario proves the durable
  68. * `tool/code-dispatch` copy of an oversized sub-result is bounded to a
  69. * preview + locator while the program value stays whole.
  70. */
  71. spillMaxInlineBytes?: number
  72. /** Run scenario-specific terminal input instead of replaying recorded user prompts. */
  73. interaction?: ScenarioInteraction
  74. /**
  75. * Mount a deterministic compaction backend plus `/compact`, then run the
  76. * human command with a held summary while a prompt and injected context
  77. * arrive. Proves queued input waits for the standalone bracket's durability
  78. * checkpoint instead of racing the replacement.
  79. */
  80. manualCompact?: boolean
  81. }
  82. const SCENARIOS: Scenario[] = [
  83. {
  84. name: 'multi-turn-conversation',
  85. composition: 'native',
  86. expectedTools: [],
  87. expectedEventCounts: { 'plan/mode': 2 },
  88. enterPlanMode: true,
  89. leavePlanModeAfterFirstTurn: true,
  90. recorded: true,
  91. },
  92. {
  93. name: 'queued-manual-compact',
  94. fixture: 'multi-turn-conversation',
  95. composition: 'native',
  96. expectedTools: [],
  97. recorded: false,
  98. manualCompact: true,
  99. },
  100. {
  101. name: 'todo-plan',
  102. composition: 'native',
  103. expectedTools: ['todo_write'],
  104. expectedEventCounts: { 'todo/write': 1 },
  105. recorded: true,
  106. enableTodo: true,
  107. },
  108. {
  109. name: 'bash-terminal-card',
  110. composition: 'native',
  111. expectedTools: ['bash'],
  112. recorded: true,
  113. },
  114. {
  115. name: 'parallel-file-reads',
  116. composition: 'native',
  117. expectedTools: ['read', 'read'],
  118. recorded: true,
  119. seedWorkspace: true,
  120. },
  121. {
  122. name: 'skill-invocation-policy',
  123. composition: 'native',
  124. expectedTools: [],
  125. recorded: false,
  126. seedWorkspace: true,
  127. interaction: 'skill-invocation-policy',
  128. },
  129. {
  130. name: 'code-mode',
  131. composition: 'code',
  132. expectedTools: ['run_code'],
  133. expectedEventCounts: { 'tool/code-dispatch': 2 },
  134. recorded: true,
  135. },
  136. {
  137. name: 'code-mode-dispatch-spill',
  138. composition: 'code',
  139. expectedTools: ['run_code'],
  140. expectedEventCounts: { 'tool/code-dispatch-start': 1, 'tool/code-dispatch': 1 },
  141. recorded: true,
  142. spillMaxInlineBytes: 600,
  143. },
  144. {
  145. name: 'dynamic-workflow',
  146. composition: 'native',
  147. expectedTools: ['workflow'],
  148. childSessions: 1,
  149. recorded: true,
  150. },
  151. {
  152. name: 'cordis-dynamic-toolchain',
  153. composition: 'advanced',
  154. expectedTools: ['cordis_mount', 'run_code', 'subagent', 'workflow', 'cordis_unmount'],
  155. expectedEventCounts: { 'tool/code-dispatch': 1 },
  156. childSessions: 2,
  157. recorded: false,
  158. },
  159. ]
  160. function snapshotModeFromEnv(value: string | undefined): SnapshotMode {
  161. if (value === undefined || value === '' || value === 'replay') return 'replay'
  162. if (value === 'record' || value === 'refresh') return value
  163. throw new Error(`DSH_SNAPSHOT must be replay, record, or refresh; got ${JSON.stringify(value)}`)
  164. }
  165. const MODE = snapshotModeFromEnv(process.env.DSH_SNAPSHOT)
  166. const observedScenarios = new Set<string>()
  167. const workerState = Reflect.get(globalThis, '__vitest_worker__') as
  168. | { readonly config?: { readonly testNamePattern?: RegExp } }
  169. | undefined
  170. // Worker argv omits the parent CLI's `-t`; the serialized runner config is the
  171. // authoritative distinction between a focused replay and the full suite.
  172. const TEST_NAME_FILTERED = workerState?.config?.testNamePattern !== undefined
  173. /**
  174. * Deterministic keyless summary that pauses so the scenario can submit a real
  175. * prompt and inject context while manual compaction holds turn admission.
  176. */
  177. class DeferredSnapshotCompactService extends BasicCompactService {
  178. readonly summaryStarted = Promise.withResolvers<undefined>()
  179. readonly releaseSummary = Promise.withResolvers<undefined>()
  180. override async summarize(
  181. _input: SummarizationInput,
  182. _agent: Agent,
  183. signal?: AbortSignal,
  184. ): Promise<{ summary: [{ type: 'text'; text: string }]; provider: string; model: string }> {
  185. this.summaryStarted.resolve(undefined)
  186. await this.releaseSummary.promise
  187. signal?.throwIfAborted()
  188. return {
  189. summary: [{ type: 'text', text: 'Keyless manual compaction checkpoint.' }],
  190. provider: 'snapshot',
  191. model: 'snapshot-compactor',
  192. }
  193. }
  194. }
  195. /** Seed between-turn model-visible history without inventing a loop execution. */
  196. function seedCompactableHistory(agent: Agent): void {
  197. agent.inject(createUserMessage({
  198. content: [{ type: 'text', text: 'Older snapshot context. '.repeat(60) }],
  199. source: { kind: 'plugin', plugin: 'snapshot-seed' },
  200. }))
  201. }
  202. function snapshotDisplayPath(displayPath: string, cwd: string, displayCwd: string): string {
  203. const rel = relative(cwd, displayPath)
  204. if (rel === '') return displayCwd
  205. if (isAbsolute(rel) || rel === '..' || rel.startsWith(`..${sep}`)) return displayPath
  206. return `${displayCwd}/${rel.split(sep).join('/')}`
  207. }
  208. function scenarioDir(scenario: Scenario): string {
  209. return join(SNAPSHOTS_DIR, scenario.name)
  210. }
  211. /** Directory owning the replay fixture: the scenario's own, or the one it derives from. */
  212. function fixtureDir(scenario: Scenario): string {
  213. return join(SNAPSHOTS_DIR, scenario.fixture ?? scenario.name)
  214. }
  215. function childFixturePaths(scenario: Scenario): string[] {
  216. return Array.from(
  217. { length: scenario.childSessions ?? 0 },
  218. (_, index) => join(fixtureDir(scenario), `session.${index + 1}.jsonl`),
  219. )
  220. }
  221. function userPrompts(rawLog: string): string[] {
  222. return parseSessionLog(rawLog).flatMap((event) => {
  223. if (event.type !== 'user/message' || event.data.source.kind !== 'user') return []
  224. const text = event.data.content
  225. .filter(block => block.type === 'text')
  226. .map(block => block.text)
  227. .join('')
  228. return text.length > 0 ? [text] : []
  229. })
  230. }
  231. function rawSessionLog(session: Session): string {
  232. return [
  233. JSON.stringify({ type: 'session', ...session.header }),
  234. ...packChunkRuns(session.events).map(record => JSON.stringify(record)),
  235. '',
  236. ].join('\n')
  237. }
  238. function normalizeTerminalSnapshot(snapshot: string, cwd: string, displayCwd: string): string {
  239. return snapshot
  240. .split(`/private${cwd}`).join('/workspace/project')
  241. .split(displayCwd).join('/workspace/project')
  242. .split(cwd).join('/workspace/project')
  243. .replace(UUID_RE, '{{uuid}}')
  244. }
  245. async function settleTerminal(terminal: HeadlessTerminal): Promise<void> {
  246. let stable = 0
  247. for (let attempt = 0; attempt < 20 && stable < 3; attempt++) {
  248. const before = terminal.frames
  249. await new Promise(resolve => setTimeout(resolve, 10))
  250. await terminal.flush()
  251. stable = terminal.frames === before ? stable + 1 : 0
  252. }
  253. if (stable < 3) throw new Error('TUI frames did not quiesce within 200ms')
  254. }
  255. /** Bound deterministic in-process coordination waits with actionable state. */
  256. async function snapshotDeadline<T>(
  257. operation: Promise<T>,
  258. detail: () => string,
  259. ): Promise<T> {
  260. let timer: ReturnType<typeof setTimeout> | undefined
  261. try {
  262. return await Promise.race([
  263. operation,
  264. new Promise<never>((_resolve, reject) => {
  265. timer = setTimeout(() => { reject(new Error(detail())) }, 5_000)
  266. }),
  267. ])
  268. } finally {
  269. if (timer !== undefined) clearTimeout(timer)
  270. }
  271. }
  272. async function mountScenarioContext(
  273. scenario: Scenario,
  274. cwd: string,
  275. displayCwd: string,
  276. fixtureFile: string,
  277. childFiles: string[],
  278. ): Promise<Context> {
  279. class SnapshotLocalFileSystem extends LocalFileSystem {
  280. override async resolve(
  281. path: string,
  282. opts?: { cwd?: string; signal?: AbortSignal },
  283. ): Promise<Awaited<ReturnType<LocalFileSystem['resolve']>>> {
  284. const target = await super.resolve(path, opts)
  285. return { ...target, displayPath: snapshotDisplayPath(target.displayPath, cwd, displayCwd) }
  286. }
  287. }
  288. const ctx = new Context()
  289. await ctx.plugin(AgentCore, {
  290. agents: [],
  291. dshHome: join(cwd, '.dsh'),
  292. workspaceContext: false,
  293. tools: { mode: scenario.composition === 'code' ? 'code' : scenario.composition === 'advanced' ? 'both' : 'native' },
  294. skills: { local: { agentsHome: join(cwd, '.agents') } },
  295. })
  296. await ctx.plugin(TokenMeterService)
  297. if (scenario.manualCompact === true) {
  298. await ctx.plugin(DeferredSnapshotCompactService, { auto: false })
  299. }
  300. await ctx.plugin(LocalSubprocessService)
  301. await ctx.plugin(LocalBashExecutor, { cwd, timeoutMs: 30_000 })
  302. await ctx.plugin(SnapshotLocalFileSystem, { cwd: '/' })
  303. await ctx.plugin(FsPolicy)
  304. await ctx.plugin(ToolFs)
  305. await ctx.plugin(UserInteractionService)
  306. await ctx.plugin(TuiPromptService)
  307. // todo_write is opt-in: only the todo-plan scenario mounts it, matching the shipped
  308. // config that omits it. The other scenarios prove the default todo-free composition.
  309. if (scenario.enableTodo === true) await ctx.plugin(ToolTodo)
  310. await ctx.plugin(SubagentService)
  311. await ctx.plugin(SubagentSpawn, { providerName: 'spawn' })
  312. await ctx.plugin(ToolSubagent, { provider: 'spawn', toolName: 'subagent', enableRunInBackground: false })
  313. await ctx.plugin(WorkerWorkflowEngine, { provider: 'spawn' })
  314. await ctx.plugin(ToolWorkflow)
  315. await ctx.plugin(ToolRalph)
  316. await ctx.plugin(CommandService)
  317. if (scenario.manualCompact === true) await ctx.plugin(CommandCompact)
  318. if (scenario.enterPlanMode === true) {
  319. await ctx.plugin(PlanModeService, { section: 'Snapshot plan mode instructions.' })
  320. }
  321. if (scenario.composition === 'code' || scenario.composition === 'advanced') {
  322. await ctx.plugin(WorkerCodeRuntime, {})
  323. }
  324. if (scenario.spillMaxInlineBytes !== undefined) {
  325. await ctx.plugin(LocalSpillStore, { root: join(cwd, '.spill') })
  326. await ctx.plugin(SpillPolicy, { maxInlineBytes: scenario.spillMaxInlineBytes })
  327. }
  328. if (scenario.composition === 'advanced') await ctx.plugin(ToolCordis, { vmTimeoutMs: 5_000 })
  329. if (MODE === 'record' && scenario.recorded) {
  330. await ctx.plugin(LlmDeepSeek)
  331. } else {
  332. installLlmReplay(ctx, { file: fixtureFile, childFiles, providers: PROVIDERS })
  333. }
  334. return ctx
  335. }
  336. interface ScenarioResult {
  337. terminal: string
  338. parent: Session
  339. children: Session[]
  340. workflowEvents: string[]
  341. }
  342. async function runScenario(scenario: Scenario): Promise<ScenarioResult> {
  343. const snapshotTime = new Date(2026, 6, 21, 12, 0, 0).getTime()
  344. const clock = vi.spyOn(Date, 'now').mockReturnValue(snapshotTime)
  345. const fixtureFile = join(fixtureDir(scenario), 'session.jsonl')
  346. const childFiles = childFixturePaths(scenario)
  347. const prompts = userPrompts(await readFile(fixtureFile, 'utf8'))
  348. if (scenario.interaction === undefined) {
  349. expect(prompts.length, `${scenario.name} must carry at least one recorded user prompt`).toBeGreaterThan(0)
  350. }
  351. const cwd = await mkdtemp(join(SNAPSHOT_TMP_ROOT, `dsh-tui-snapshot-${scenario.name}-`))
  352. const displayCwd = `/tmp/${basename(cwd)}`
  353. let ctx: Context | undefined
  354. let controller: ReturnType<typeof createTuiChat> | undefined
  355. const terminal = new HeadlessTerminal(100, 36)
  356. try {
  357. if (scenario.seedWorkspace === true) {
  358. const source = join(fixtureDir(scenario), 'workspace')
  359. await cp(source, cwd, { recursive: true })
  360. }
  361. ctx = await mountScenarioContext(scenario, cwd, displayCwd, fixtureFile, childFiles)
  362. const disposedSessions: Session[] = []
  363. ctx.on('session/disposed', (session) => { disposedSessions.push(session) })
  364. const workflowEvents: string[] = []
  365. for (const name of ['workflow/start', 'workflow/phase', 'workflow/agent-start', 'workflow/agent-end', 'workflow/end'] as const) {
  366. ctx.on(name, () => { workflowEvents.push(name) })
  367. }
  368. const handle = await ctx.agents.create({
  369. sessionId: SessionId('main-session'),
  370. meta: { cwd },
  371. agentOptions: { provider: 'deepseek-official', model: 'deepseek-v4-flash' },
  372. })
  373. const agent: Agent = handle.agent
  374. if (scenario.manualCompact === true) seedCompactableHistory(agent)
  375. controller = createTuiChat(ctx, {
  376. sessionId: 'main-session',
  377. theme: { color: true },
  378. showReasoning: true,
  379. title: 'DSH TUI snapshot',
  380. welcome: `Recorded replay: ${scenario.name}`,
  381. maxToolOutputLines: 8,
  382. }, {
  383. terminal,
  384. exit: () => {},
  385. formatCwd: () => displayCwd,
  386. })
  387. await settleTerminal(terminal)
  388. let interactionSnapshot: string | undefined
  389. if (scenario.interaction === 'skill-invocation-policy') {
  390. terminal.send('/skill')
  391. await settleTerminal(terminal)
  392. const discovery = normalizeTerminalSnapshot(
  393. await terminal.snapshot({ includeScrollback: true }),
  394. cwd,
  395. displayCwd,
  396. )
  397. expect(discovery).toContain('user-only-skill')
  398. expect(discovery).not.toContain('model-only-skill')
  399. terminal.send('\x03')
  400. await settleTerminal(terminal)
  401. const skillContext = ctx
  402. const skillTurnEnded = new Promise<void>((resolve) => {
  403. const detach = skillContext.on('session/event', (session, event) => {
  404. if (session !== agent.session || event.type !== 'turn/end') return
  405. detach()
  406. resolve()
  407. })
  408. })
  409. terminal.send('/skill:user-only-skill')
  410. terminal.send('\r')
  411. await skillTurnEnded
  412. await agent.whenIdle()
  413. await settleTerminal(terminal)
  414. const loaded = normalizeTerminalSnapshot(
  415. await terminal.snapshot({ includeScrollback: true }),
  416. cwd,
  417. displayCwd,
  418. )
  419. expect(loaded).toContain('USER-ONLY SKILL LOADED')
  420. terminal.send('/skill:model-only-skill')
  421. terminal.send('\r')
  422. await settleTerminal(terminal)
  423. const denied = normalizeTerminalSnapshot(
  424. await terminal.snapshot({ includeScrollback: true }),
  425. cwd,
  426. displayCwd,
  427. )
  428. expect(denied).toContain('model-only-skill')
  429. expect(denied).toContain('not available for user invocation.')
  430. expect(denied).not.toContain('MODEL-ONLY BODY MUST NOT LOAD')
  431. interactionSnapshot = [
  432. '=== skill autocomplete ===',
  433. discovery,
  434. '',
  435. '=== loaded exact invocation ===',
  436. loaded,
  437. '',
  438. '=== denied exact invocation ===',
  439. denied,
  440. ].join('\n')
  441. }
  442. let remainingPrompts = prompts
  443. let queuedPrompt: string | undefined
  444. let manualOrder: string[] | undefined
  445. let manualCommandId: string | undefined
  446. if (scenario.manualCompact === true) {
  447. expect(prompts.length, 'queued manual compaction needs a second replayed prompt').toBeGreaterThanOrEqual(2)
  448. queuedPrompt = prompts.at(-1)
  449. remainingPrompts = prompts.slice(0, -1)
  450. }
  451. if (scenario.enterPlanMode === true) {
  452. const firstPrompt = prompts[0]!
  453. terminal.send(`/plan ${firstPrompt}`)
  454. terminal.send('\r')
  455. await agent.whenIdle()
  456. await settleTerminal(terminal)
  457. remainingPrompts = prompts.slice(1)
  458. }
  459. if (scenario.leavePlanModeAfterFirstTurn === true) {
  460. terminal.send('/plan off')
  461. terminal.send('\r')
  462. await settleTerminal(terminal)
  463. }
  464. for (const prompt of remainingPrompts) {
  465. const admitted = agent.session.events.filter(event =>
  466. event.type === 'user/message' && event.data.source.kind === 'user').length
  467. terminal.send(prompt)
  468. terminal.send('\r')
  469. await terminal.flush()
  470. await expect.poll(() => agent.session.events.filter(event =>
  471. event.type === 'user/message' && event.data.source.kind === 'user').length).toBe(admitted + 1)
  472. await agent.whenIdle()
  473. await settleTerminal(terminal)
  474. }
  475. if (scenario.manualCompact === true && queuedPrompt !== undefined) {
  476. terminal.send('/help')
  477. terminal.send('\r')
  478. await settleTerminal(terminal)
  479. expect(await terminal.snapshot({ includeScrollback: true }))
  480. .toContain('/compact — Compact older conversation history')
  481. const compact = ctx.compact as DeferredSnapshotCompactService
  482. const inbox: string[] = []
  483. manualOrder = []
  484. ctx.on('agent/inbox/enqueue', (subject, item) => {
  485. if (subject === agent) inbox.push(`enqueue:${item.placement}:${item.id}`)
  486. })
  487. ctx.on('agent/inbox/dequeue', (subject, message) => {
  488. if (subject === agent) inbox.push(`dequeue:${message.id}`)
  489. })
  490. ctx.on('session/event', (session, event) => {
  491. if (session !== agent.session) return
  492. if (event.type === 'command/run' && event.data.name === 'compact') {
  493. manualCommandId = event.data.commandId
  494. manualOrder?.push('command/run')
  495. }
  496. if (event.type === 'command/done' && event.data.commandId === manualCommandId) {
  497. manualOrder?.push('command/done')
  498. }
  499. if (event.type.startsWith('compact/')) manualOrder?.push(event.type)
  500. if (event.type === 'user/message'
  501. && event.data.source.kind === 'plugin'
  502. && event.data.source.plugin === 'compact') manualOrder?.push('checkpoint')
  503. if (event.type === 'turn/start') manualOrder?.push(`turn/start:${event.data.trigger.kind}`)
  504. })
  505. ctx.on('session/flush', (session) => {
  506. if (session === agent.session) manualOrder?.push('flush')
  507. })
  508. terminal.send('/compact')
  509. terminal.send('\r')
  510. await terminal.flush()
  511. await snapshotDeadline(compact.summaryStarted.promise, () =>
  512. `manual summary did not start; status=${agent.status}; tail=${
  513. agent.session.events.slice(-8).map(event => event.type).join(',')
  514. }`)
  515. clock.mockReturnValue(snapshotTime + 1_000)
  516. await settleTerminal(terminal)
  517. await expect.poll(() => terminal.snapshot()).toContain('dsh ⊙')
  518. await expect.poll(() => terminal.snapshot()).toContain('Context being compacted 1.0s')
  519. const liveCompaction = await terminal.snapshot()
  520. expect(liveCompaction.indexOf('Context being compacted 1.0s')).toBeLessThan(liveCompaction.indexOf('dsh ⊙'))
  521. clock.mockReturnValue(snapshotTime)
  522. // Real keystrokes: the prompt keeps its ordinary queue identity while
  523. // admission is reserved, and an injection appends immediately.
  524. terminal.send(queuedPrompt)
  525. terminal.send('\r')
  526. await terminal.flush()
  527. await expect.poll(() => inbox.length).toBe(1)
  528. agent.inject(createUserMessage({
  529. content: [{ type: 'text', text: 'Injected while compaction was running.' }],
  530. source: { kind: 'plugin', plugin: 'snapshot-injector' },
  531. }))
  532. expect(inbox[0]).toMatch(/^enqueue:queued:/u)
  533. expect(agent.status).toBe('idle')
  534. expect(agent.session.events.some(event => event.type === 'user/message'
  535. && event.data.source.kind === 'user'
  536. && event.data.content.some(block => block.type === 'text' && block.text === queuedPrompt))).toBe(false)
  537. const idle = agent.whenIdle()
  538. compact.releaseSummary.resolve(undefined)
  539. await snapshotDeadline(idle, () =>
  540. `manual compaction did not reach idle; status=${agent.status}; order=${manualOrder?.join(',') ?? ''}; tail=${
  541. agent.session.events.slice(-12).map(event => event.type).join(',')
  542. }`)
  543. await settleTerminal(terminal)
  544. expect(inbox).toEqual([inbox[0], `dequeue:${inbox[0]?.slice('enqueue:queued:'.length) ?? ''}`])
  545. }
  546. const events: SessionEvent[] = [...agent.session.events]
  547. const firstHeader = events.find(event => event.type === 'request/header')
  548. expect(firstHeader?.type === 'request/header' && firstHeader.data.header.system)
  549. .toContain(FILE_REFERENCE_PROMPT)
  550. expect(events.filter(event => event.type === 'tool/call').map(event => event.data.name)).toEqual(scenario.expectedTools)
  551. for (const [type, count] of Object.entries(scenario.expectedEventCounts ?? {})) {
  552. expect(events.filter(event => event.type === type), `${scenario.name} must emit ${type}`).toHaveLength(count)
  553. }
  554. if (scenario.enterPlanMode === true) {
  555. expect(ctx.planMode.get(agent)).toEqual({
  556. active: scenario.leavePlanModeAfterFirstTurn !== true,
  557. })
  558. const planMode = events.find(event => event.type === 'plan/mode')
  559. if (planMode === undefined || firstHeader === undefined) {
  560. throw new Error('plan-mode command snapshot needs plan/mode before its first request/header')
  561. }
  562. expect(planMode.seq).toBeLessThan(firstHeader.seq)
  563. expect(firstHeader.data.header.system).toContain('Snapshot plan mode instructions.')
  564. const firstMessage = events.find(event => event.type === 'user/message')
  565. expect(firstMessage?.data.content).toEqual([{ type: 'text', text: prompts[0] }])
  566. }
  567. if (scenario.leavePlanModeAfterFirstTurn === true) {
  568. const planModes = events.filter(event => event.type === 'plan/mode')
  569. expect(planModes.map(event => event.data.active)).toEqual([true, false])
  570. const headers = events.filter(event => event.type === 'request/header')
  571. const exit = planModes[1]
  572. const afterExit = headers[1]
  573. if (exit === undefined || afterExit === undefined) {
  574. throw new Error('active plan exit snapshot needs a committed exit and changed request header')
  575. }
  576. expect(exit.seq).toBeLessThan(afterExit.seq)
  577. expect(afterExit.data.header.system).not.toContain('Snapshot plan mode instructions.')
  578. expect(events.filter(event => event.type === 'user/message' && event.data.source.kind === 'plugin').map(event => (event.data as { content: unknown }).content))
  579. .toContainEqual([{ type: 'text', text: 'The user switched this session back to the default mode.' }])
  580. }
  581. if (scenario.manualCompact === true) {
  582. const compactStart = events.find(event => event.type === 'compact/start')
  583. const compactSummary = events.find(event => event.type === 'compact/summary')
  584. const compactCheckpoint = events.find(event => event.type === 'user/message'
  585. && event.data.source.kind === 'plugin' && event.data.source.plugin === 'compact')
  586. const injectedEvent = events.find(event => event.type === 'user/message'
  587. && event.data.source.kind === 'plugin' && event.data.source.plugin === 'snapshot-injector')
  588. const compactEnd = events.find(event => event.type === 'compact/end')
  589. expect(compactStart?.data.turn).toBeNull()
  590. expect(compactEnd?.data.turn).toBeNull()
  591. expect(events.filter(event => event.type === 'compact/summary')).toHaveLength(1)
  592. if (compactStart === undefined || compactSummary === undefined
  593. || compactCheckpoint === undefined || injectedEvent === undefined
  594. || compactEnd === undefined) {
  595. throw new Error('manual compaction snapshot is missing its durable marker, summary, checkpoint, or injection')
  596. }
  597. // The markers are time points, not an exclusive container: unrelated
  598. // idle injection is allowed between them while the selected span stays stable.
  599. expect(compactStart.seq).toBeLessThan(injectedEvent.seq)
  600. expect(injectedEvent.seq).toBeLessThan(compactSummary.seq)
  601. expect(compactSummary.seq).toBeLessThan(compactCheckpoint.seq)
  602. expect(compactCheckpoint.seq).toBeLessThan(compactEnd.seq)
  603. const manualTimeline = manualOrder ?? []
  604. const commandRunIndex = manualTimeline.indexOf('command/run')
  605. const compactStartIndex = manualTimeline.indexOf('compact/start')
  606. const compactEndIndex = manualTimeline.indexOf('compact/end')
  607. const firstFlushIndex = manualTimeline.indexOf('flush')
  608. const queuedTurnIndex = manualTimeline.indexOf('turn/start:message')
  609. const commandDoneIndex = manualTimeline.indexOf('command/done')
  610. expect(manualTimeline.filter(item => item === 'command/run')).toHaveLength(1)
  611. expect(manualTimeline.filter(item => item === 'command/done')).toHaveLength(1)
  612. expect(compactStartIndex).toBeGreaterThan(commandRunIndex)
  613. expect(compactEndIndex).toBeGreaterThan(compactStartIndex)
  614. expect(firstFlushIndex).toBeGreaterThan(compactEndIndex)
  615. expect(queuedTurnIndex).toBeGreaterThan(firstFlushIndex)
  616. expect(commandDoneIndex).toBeGreaterThan(firstFlushIndex)
  617. const commandRun = events.find(event => event.type === 'command/run'
  618. && event.data.name === 'compact')
  619. const commandRunId = commandRun?.type === 'command/run'
  620. ? commandRun.data.commandId
  621. : undefined
  622. const commandDone = events.find(event => event.type === 'command/done'
  623. && event.data.commandId === commandRunId)
  624. expect(commandRun?.type === 'command/run' && commandRun.data).toEqual({
  625. commandId: commandRunId,
  626. name: 'compact',
  627. args: '',
  628. source: { kind: 'user' },
  629. })
  630. expect(commandDone?.type === 'command/done' && commandDone.data).toEqual({
  631. commandId: commandRunId,
  632. kind: 'success',
  633. text: 'Compacted 2 history items (~387 tokens).',
  634. })
  635. expect(commandRun !== undefined && commandRun.seq < compactStart.seq).toBe(true)
  636. expect(commandDone !== undefined && commandDone.seq > compactEnd.seq).toBe(true)
  637. expect(agent.session.surface.nodes).not.toContain(commandRun?.seq)
  638. expect(agent.session.surface.nodes).not.toContain(commandDone?.seq)
  639. // The command line itself never becomes a prompt.
  640. expect(events.some(event => event.type === 'user/message'
  641. && event.data.source.kind === 'user'
  642. && event.data.content.some(block => block.type === 'text' && block.text.trim() === '/compact'))).toBe(false)
  643. const derived = agent.session.deriveMessages().map(message => message.content
  644. .map(block => block.type === 'text' ? block.text : '')
  645. .join(''))
  646. const checkpoint = derived.findIndex(text => text.includes('Keyless manual compaction checkpoint.'))
  647. const injected = derived.findIndex(text => text.includes('Injected while compaction was running.'))
  648. const queued = derived.findIndex(text => text === queuedPrompt)
  649. expect(checkpoint).toBe(0)
  650. expect(injected).toBeGreaterThan(checkpoint)
  651. expect(queued).toBeGreaterThan(injected)
  652. expect(derived).not.toContain('/compact')
  653. expect(derived).not.toContain('Compacted 2 history items (~387 tokens).')
  654. expect(derived.filter(text => text.includes('Injected while compaction was running.'))).toHaveLength(1)
  655. expect(compactSummary.data.shadowedSeqs).not.toContain(injectedEvent.seq)
  656. const queuedTurn = events.findLast(event => event.type === 'turn/start')
  657. expect(queuedTurn !== undefined && compactEnd.seq < queuedTurn.seq).toBe(true)
  658. }
  659. if (scenario.spillMaxInlineBytes !== undefined) {
  660. // The REAL pipeline ran (tools execute on replay too): the durable
  661. // dispatch copy is bounded to a preview + locator under the run cwd,
  662. // while the outer result still carries the program's whole value.
  663. const dispatch = events.find(event => (event.type as string) === 'tool/code-dispatch')
  664. const content = (dispatch?.data as { content: { type: string; text?: string }[] }).content
  665. const text = content.filter(block => block.type === 'text').map(block => block.text ?? '').join('')
  666. expect(Buffer.byteLength(text, 'utf8')).toBeLessThanOrEqual(scenario.spillMaxInlineBytes)
  667. expect(text).toContain('Full formatted result stored at:')
  668. expect(text).toContain('.spill')
  669. }
  670. expect(events.filter(event => event.type === 'tool/result').every(event => !event.data.message.content[0].isError)).toBe(true)
  671. expect(events.filter(event => event.type === 'turn/end').every(event => event.data.reason.kind !== 'error')).toBe(true)
  672. if (scenario.name === 'dynamic-workflow' || scenario.name === 'cordis-dynamic-toolchain') {
  673. expect(workflowEvents).toEqual([
  674. 'workflow/start',
  675. 'workflow/phase',
  676. 'workflow/agent-start',
  677. 'workflow/agent-end',
  678. 'workflow/end',
  679. ])
  680. }
  681. expect(terminal.themeViolations(), `${scenario.name} must remain theme-agnostic`).toEqual([])
  682. const snapshot = interactionSnapshot ?? normalizeTerminalSnapshot(
  683. await terminal.snapshot({ includeScrollback: true }),
  684. cwd,
  685. displayCwd,
  686. )
  687. await handle.dispose()
  688. const children = disposedSessions
  689. .filter(session => session !== agent.session)
  690. .sort((a, b) => a.header.createdAt - b.header.createdAt)
  691. expect(children).toHaveLength(scenario.childSessions ?? 0)
  692. return { terminal: snapshot, parent: agent.session, children, workflowEvents }
  693. } finally {
  694. await controller?.dispose()
  695. await ctx?.fiber.dispose()
  696. await terminal.dispose()
  697. await rm(cwd, { recursive: true, force: true })
  698. clock.mockRestore()
  699. }
  700. }
  701. async function writeRecording(scenario: Scenario, result: ScenarioResult): Promise<void> {
  702. const dir = scenarioDir(scenario)
  703. await mkdir(dir, { recursive: true })
  704. await writeFile(
  705. join(dir, 'session.jsonl'),
  706. scrubRequestHeaders(tokenizeSessionFixtureCwd(rawSessionLog(result.parent))),
  707. )
  708. expect(result.children).toHaveLength(scenario.childSessions ?? 0)
  709. for (const [index, child] of result.children.entries()) {
  710. await writeFile(
  711. join(dir, `session.${index + 1}.jsonl`),
  712. scrubRequestHeaders(tokenizeSessionFixtureCwd(rawSessionLog(child))),
  713. )
  714. }
  715. }
  716. describe('TUI recorded-session terminal snapshots', () => {
  717. for (const scenario of SCENARIOS) {
  718. it(scenario.name, async () => {
  719. observedScenarios.add(scenario.name)
  720. const result = await runScenario(scenario)
  721. const terminalFile = join(scenarioDir(scenario), 'terminal.expected.txt')
  722. if (MODE === 'record' || MODE === 'refresh') {
  723. await mkdir(scenarioDir(scenario), { recursive: true })
  724. await writeFile(terminalFile, result.terminal)
  725. }
  726. if (MODE === 'record' && scenario.recorded) await writeRecording(scenario, result)
  727. await expect(result.terminal).toMatchFileSnapshot(terminalFile)
  728. }, 120_000)
  729. }
  730. })
  731. afterAll(async () => {
  732. const scenarioNames = SCENARIOS.map(scenario => scenario.name).sort()
  733. const observedNames = [...observedScenarios].sort()
  734. if (TEST_NAME_FILTERED) {
  735. expect(observedNames).not.toHaveLength(0)
  736. expect(scenarioNames).toEqual(expect.arrayContaining(observedNames))
  737. } else {
  738. expect(observedNames).toEqual(scenarioNames)
  739. }
  740. for (const [index, scenario] of SCENARIOS.entries()) {
  741. if (scenario.fixture === undefined) continue
  742. const sourceIndex = SCENARIOS.findIndex(candidate => candidate.name === scenario.fixture)
  743. expect(sourceIndex, `${scenario.name} fixture source ${scenario.fixture} must exist`).toBeGreaterThanOrEqual(0)
  744. expect(sourceIndex, `${scenario.name} fixture source must precede it`).toBeLessThan(index)
  745. const source = SCENARIOS[sourceIndex]
  746. expect(source?.fixture, `${scenario.name} fixture source must own its replay files`).toBeUndefined()
  747. expect(source?.recorded, `${scenario.name} fixture source must be recordable`).toBe(true)
  748. }
  749. const directories = (await readdir(SNAPSHOTS_DIR, { withFileTypes: true }))
  750. .filter(entry => entry.isDirectory())
  751. .map(entry => entry.name)
  752. .sort()
  753. expect(directories).toEqual(SCENARIOS.map(scenario => scenario.name).sort())
  754. for (const scenario of SCENARIOS) {
  755. const expected = [
  756. ...scenario.fixture === undefined ? ['session.jsonl'] : [],
  757. 'terminal.expected.txt',
  758. ...scenario.seedWorkspace === true && scenario.fixture === undefined ? ['workspace'] : [],
  759. ...Array.from({ length: scenario.childSessions ?? 0 }, (_, index) => `session.${index + 1}.jsonl`),
  760. ].sort()
  761. expect((await readdir(scenarioDir(scenario))).sort()).toEqual(expected)
  762. for (const fixture of ['session.jsonl', ...childFixturePaths(scenario).map(path => basename(path))]) {
  763. const content = await readFile(join(fixtureDir(scenario), fixture), 'utf8')
  764. expect(scrubRequestHeaders(content), `${scenario.name}/${fixture} carries request-header bulk`).toBe(content)
  765. }
  766. }
  767. })