suite.spec.ts 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595
  1. import { cpSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
  2. import { rm } from 'node:fs/promises'
  3. import { tmpdir } from 'node:os'
  4. import { join } from 'node:path'
  5. import { fileURLToPath } from 'node:url'
  6. import { afterAll, describe, expect, it } from 'vitest'
  7. import { defineAcpSnapshotSuite, type HarvestedLog, type Scenario } from '../src/index.ts'
  8. import {
  9. fixtureContext,
  10. formatSystemPromptSnapshot,
  11. headerChangeCount,
  12. formatToolSchemasSnapshot,
  13. normalizedHeaders,
  14. normalizedSystemPrompts,
  15. normalizedToolSchemas,
  16. parseToolSchemasSnapshot,
  17. refreshFixtureReplacements,
  18. scenarioSkipped,
  19. sessionFixtureNames,
  20. restorePinnedToolSchemas,
  21. stabilizeRefreshLog,
  22. stdoutExpectedVariants,
  23. unknownToolCallIds,
  24. } from '../src/suite.ts'
  25. /**
  26. * Unit tests for the suite factory, by running it: two synthetic suites over the scripted fake
  27. * ACP bin (./fixtures/fake-acp-agent.ts) register real describe/it trees at collection time,
  28. * so every factory path — expected-output and log comparisons, the per-suite header pin and its uniformity
  29. * guard, record-mode fixture write-back, skip semantics, and the fixture guard block —
  30. * executes as an ordinary green test.
  31. *
  32. * Record tests use a temp copy. To intentionally rebuild their committed fixtures, run this
  33. * spec once with `ACP_SNAPSHOT_SPEC_BOOTSTRAP=1`, then review and commit the resulting tree.
  34. */
  35. const fakeAgent = fileURLToPath(new URL('./fixtures/fake-acp-agent.ts', import.meta.url))
  36. const AGENT = {
  37. binScript: fakeAgent,
  38. libBinScript: fakeAgent,
  39. configPath: fakeAgent,
  40. tsconfigPath: fileURLToPath(new URL('../../../../tsconfig.json', import.meta.url)),
  41. }
  42. const REPLAY_DIR = fileURLToPath(new URL('./fixtures/suite', import.meta.url))
  43. const RECORD_SRC = fileURLToPath(new URL('./fixtures/record-suite', import.meta.url))
  44. // Replay pins explicit header classes; recording covers the default fallback.
  45. const REPLAY_SCENARIOS: Scenario[] = [
  46. { name: 'pin-turn', hasModelTurn: true, recorded: true, pinsHeader: true, expectedHeaderChanges: 1, headerClass: 'main' },
  47. {
  48. name: 'plain-turn',
  49. hasModelTurn: true,
  50. recorded: true,
  51. headerClass: 'main',
  52. env: { DSH_PERMISSION_MODE: 'never' },
  53. configPath: AGENT.configPath,
  54. workspaceParent: tmpdir(),
  55. },
  56. { name: 'no-model', hasModelTurn: false, recorded: false, headerClass: 'main' },
  57. { name: 'blocked-log', hasModelTurn: false, comparesLog: true, recorded: false, headerClass: 'main' },
  58. { name: 'authored-error', hasModelTurn: true, recorded: false, overridden: true, headerClass: 'main' },
  59. ]
  60. const RECORD_SCENARIOS: Scenario[] = [
  61. { name: 'rec-pin', hasModelTurn: true, recorded: true, pinsHeader: true },
  62. { name: 'rec-child', hasModelTurn: true, recorded: true },
  63. // recorded:false in record mode → registered but skipped (never re-recorded).
  64. { name: 'rec-skip', hasModelTurn: true, recorded: false, overridden: true },
  65. ]
  66. // Record/refresh modes mutate their snapshots dir, so run them on throwaway
  67. // copies — except record's documented bootstrap knob, which regenerates the
  68. // committed record fixtures and expected outputs in place.
  69. const BOOTSTRAP = process.env.ACP_SNAPSHOT_SPEC_BOOTSTRAP === '1'
  70. const recordDir = BOOTSTRAP ? RECORD_SRC : mkdtempSync(join(tmpdir(), 'acp-snap-record-suite-'))
  71. if (!BOOTSTRAP) {
  72. cpSync(RECORD_SRC, recordDir, { recursive: true })
  73. // Record mode owns its output inventory: a new scenario has no primary yet,
  74. // while a changed child count can leave old numbered fixtures behind.
  75. rmSync(join(recordDir, 'rec-pin', 'session.jsonl'))
  76. writeFileSync(join(recordDir, 'rec-child', 'session.2.jsonl'), 'stale child\n')
  77. }
  78. const refreshDir = mkdtempSync(join(tmpdir(), 'acp-snap-refresh-suite-'))
  79. cpSync(REPLAY_DIR, refreshDir, { recursive: true })
  80. staleRefreshFixtures(refreshDir)
  81. afterAll(async () => {
  82. if (!BOOTSTRAP) await rm(recordDir, { recursive: true, force: true })
  83. await rm(refreshDir, { recursive: true, force: true })
  84. })
  85. function staleRefreshFixtures(dir: string): void {
  86. writeFileSync(join(dir, 'plain-turn', 'stdout.expected.jsonl'), 'stale stdout\n')
  87. writeFileSync(join(dir, 'pin-turn', 'system-prompt.expected.md'), 'STALE PROMPT\n')
  88. writeFileSync(join(dir, 'pin-turn', 'tool-schemas.expected.json'), '{"initial":[{"name":"stale"}],"changes":[]}\n')
  89. const plainBehaviorFile = join(dir, 'plain-turn', 'behavior.json')
  90. const plainBehavior = JSON.parse(readFileSync(plainBehaviorFile, 'utf8')) as Record<string, unknown>
  91. plainBehavior.echoEnv = true
  92. writeFileSync(plainBehaviorFile, `${JSON.stringify(plainBehavior, null, 2)}\n`)
  93. writeFileSync(join(dir, 'blocked-log', 'session.jsonl'), [
  94. '{"type":"session","id":"99999999-8888-4777-8666-555555555555","createdAt":13,"cwd":"/rec/blocked-cwd","delegationDepth":0}',
  95. '{"type":"hook/result","seq":1,"time":13,"data":{"decision":"stale","durationMs":99}}',
  96. '',
  97. ].join('\n'))
  98. writeFileSync(join(dir, 'authored-error', 'session.jsonl'), [
  99. '{"type":"session","id":"77777777-8888-4777-8666-555555555555","createdAt":13,"cwd":"/rec/error-cwd","delegationDepth":0}',
  100. '{"type":"turn/end","seq":1,"time":9,"data":{"error":"stale"}}',
  101. '',
  102. ].join('\n'))
  103. }
  104. describe('defineAcpSnapshotSuite: replay mode', () => {
  105. defineAcpSnapshotSuite({ agent: AGENT, snapshotsDir: REPLAY_DIR, scenarios: REPLAY_SCENARIOS, mode: 'replay' })
  106. })
  107. // The record suite's tests run in registration order: rec-pin re-records the
  108. // pinned fixture FIRST, so rec-child's uniformity guard reads the fresh pin.
  109. describe('defineAcpSnapshotSuite: record mode', () => {
  110. defineAcpSnapshotSuite({ agent: AGENT, snapshotsDir: recordDir, scenarios: RECORD_SCENARIOS, mode: 'record' })
  111. })
  112. describe('defineAcpSnapshotSuite: refresh mode', () => {
  113. defineAcpSnapshotSuite({ agent: AGENT, snapshotsDir: refreshDir, scenarios: REPLAY_SCENARIOS, mode: 'refresh' })
  114. })
  115. describe('defineAcpSnapshotSuite: refresh write-back', () => {
  116. it('rewrites stdout and comparable logs from a replay-mode child run', () => {
  117. const stdout = readFileSync(join(refreshDir, 'plain-turn', 'stdout.expected.jsonl'), 'utf8')
  118. expect(stdout).not.toContain('stale stdout')
  119. expect(stdout).toContain('env:{\\"mode\\":\\"replay\\"')
  120. expect(stdout).not.toContain('\\"mode\\":\\"refresh\\"')
  121. // The scenario's own env layer reached the subprocess.
  122. expect(stdout).toContain('\\"permissionMode\\":\\"never\\"')
  123. const blocked = readFileSync(join(refreshDir, 'blocked-log', 'session.jsonl'), 'utf8')
  124. expect(blocked).toContain('"decision":"block"')
  125. expect(blocked).not.toContain('"decision":"stale"')
  126. const authored = readFileSync(join(refreshDir, 'authored-error', 'session.jsonl'), 'utf8')
  127. expect(authored).toContain('"error":"model exploded"')
  128. expect(authored).not.toContain('"error":"stale"')
  129. expect(readFileSync(join(refreshDir, 'pin-turn', 'system-prompt.expected.md'), 'utf8')).toBe([
  130. 'SYS PROMPT',
  131. '',
  132. '<!-- request/header change 1 -->',
  133. '',
  134. 'SYS PROMPT',
  135. '',
  136. 'NEW PROMPT LINE',
  137. '',
  138. ].join('\n'))
  139. const schemas = readFileSync(join(refreshDir, 'pin-turn', 'tool-schemas.expected.json'), 'utf8')
  140. expect(schemas).toContain('"description": "D1"')
  141. expect(schemas).not.toContain('"name":"stale"')
  142. })
  143. })
  144. describe('defineAcpSnapshotSuite: record inventory write-back', () => {
  145. it('creates a missing primary fixture and prunes stale child fixtures', () => {
  146. expect(readFileSync(join(recordDir, 'rec-pin', 'session.jsonl'), 'utf8')).toContain('"type":"session"')
  147. expect(() => readFileSync(join(recordDir, 'rec-child', 'session.2.jsonl'), 'utf8')).toThrow()
  148. })
  149. })
  150. describe('defineAcpSnapshotSuite: registration contract', () => {
  151. it("throws when a scenario's header class has no pinning scenario", () => {
  152. expect(() => {
  153. defineAcpSnapshotSuite({
  154. agent: AGENT,
  155. snapshotsDir: REPLAY_DIR,
  156. scenarios: [{ name: 'pinless', hasModelTurn: true, recorded: true }],
  157. mode: 'replay',
  158. })
  159. }).toThrow(/no scenario pins the request-header content of class "default"/)
  160. // A pinned class does not cover a DIFFERENT class's members.
  161. expect(() => {
  162. defineAcpSnapshotSuite({
  163. agent: AGENT,
  164. snapshotsDir: REPLAY_DIR,
  165. scenarios: [
  166. { name: 'pinned', hasModelTurn: true, recorded: true, pinsHeader: true },
  167. { name: 'classless-orphan', hasModelTurn: true, recorded: true, headerClass: 'other' },
  168. ],
  169. mode: 'replay',
  170. })
  171. }).toThrow(/class "other" \(needed by classless-orphan\)/)
  172. })
  173. it('throws when two scenarios pin the same header class', () => {
  174. expect(() => {
  175. defineAcpSnapshotSuite({
  176. agent: AGENT,
  177. snapshotsDir: REPLAY_DIR,
  178. scenarios: [
  179. { name: 'first-pin', hasModelTurn: true, recorded: true, pinsHeader: true },
  180. { name: 'second-pin', hasModelTurn: true, recorded: true, pinsHeader: true },
  181. ],
  182. mode: 'replay',
  183. })
  184. }).toThrow(/header class "default" pinned by both first-pin and second-pin/)
  185. })
  186. })
  187. describe('sessionFixtureNames', () => {
  188. it('orders the primary and contiguous child fixtures while ignoring other files', () => {
  189. expect(sessionFixtureNames([
  190. 'stdout.expected.jsonl',
  191. 'session.2.jsonl',
  192. 'session.jsonl',
  193. 'session.1.jsonl',
  194. 'input.json',
  195. ])).toEqual(['session.jsonl', 'session.1.jsonl', 'session.2.jsonl'])
  196. })
  197. it('accepts a primary-only scenario', () => {
  198. expect(sessionFixtureNames(['session.jsonl'])).toEqual(['session.jsonl'])
  199. })
  200. it('rejects a directory without the primary fixture', () => {
  201. expect(() => sessionFixtureNames(['session.1.jsonl'])).toThrow('missing session.jsonl')
  202. })
  203. it('rejects gapped child fixtures', () => {
  204. expect(() => sessionFixtureNames(['session.jsonl', 'session.2.jsonl']))
  205. .toThrow('expected session.1.jsonl, found session.2.jsonl')
  206. })
  207. it.each(['session.0.jsonl', 'session.child.jsonl', 'session.01.jsonl'])(
  208. 'rejects invalid child fixture name %s',
  209. (name) => {
  210. expect(() => sessionFixtureNames(['session.jsonl', name]))
  211. .toThrow(`invalid child session fixture name: ${name}`)
  212. },
  213. )
  214. it('rejects duplicate child indexes', () => {
  215. expect(() => sessionFixtureNames(['session.jsonl', 'session.1.jsonl', 'session.1.jsonl']))
  216. .toThrow('expected session.2.jsonl, found session.1.jsonl')
  217. })
  218. })
  219. describe('stdoutExpectedVariants', () => {
  220. const scenario: Scenario = {
  221. name: 'windows-native',
  222. hasModelTurn: true,
  223. recorded: true,
  224. pinsNativeWindowsStdout: true,
  225. }
  226. it('adds the native sidecar after the shared golden on Windows', () => {
  227. expect(stdoutExpectedVariants(scenario, 'win32')).toEqual([
  228. { file: 'stdout.expected.jsonl', cwdPathMode: 'canonical' },
  229. { file: 'stdout.expected.windows.jsonl', cwdPathMode: 'native' },
  230. ])
  231. })
  232. it('keeps only the shared golden on other platforms or without the declaration', () => {
  233. expect(stdoutExpectedVariants(scenario, 'linux')).toEqual([
  234. { file: 'stdout.expected.jsonl', cwdPathMode: 'canonical' },
  235. ])
  236. expect(stdoutExpectedVariants({ ...scenario, pinsNativeWindowsStdout: false }, 'win32')).toEqual([
  237. { file: 'stdout.expected.jsonl', cwdPathMode: 'canonical' },
  238. ])
  239. })
  240. })
  241. describe('scenarioSkipped', () => {
  242. const authored: Scenario = { name: 'authored', hasModelTurn: true, recorded: false }
  243. const posix: Scenario = { name: 'posix-cancel', hasModelTurn: true, recorded: false, posixOnly: true }
  244. it('skips authored scenarios only while recording', () => {
  245. expect(scenarioSkipped(authored, true, 'linux')).toBe(true)
  246. expect(scenarioSkipped(authored, false, 'linux')).toBe(false)
  247. })
  248. it('skips posixOnly scenarios on Windows and nowhere else', () => {
  249. expect(scenarioSkipped(posix, false, 'win32')).toBe(true)
  250. expect(scenarioSkipped(posix, false, 'linux')).toBe(false)
  251. expect(scenarioSkipped(posix, false, 'darwin')).toBe(false)
  252. expect(scenarioSkipped(authored, false, 'win32')).toBe(false)
  253. })
  254. })
  255. describe('fixtureContext', () => {
  256. it('reads the fixture header id and cwd', () => {
  257. const ctx = fixtureContext('{"type":"session","id":"abc","cwd":"/rec"}\n{"type":"turn/start"}\n')
  258. expect(ctx).toEqual({ sessionIds: ['abc'], cwd: '/rec' })
  259. })
  260. it('yields no session ids for a header without a string id', () => {
  261. expect(fixtureContext('{"type":"session","cwd":"/rec"}\n').sessionIds).toEqual([])
  262. })
  263. it('falls back to an impossible sentinel cwd (never the empty string)', () => {
  264. const ctx = fixtureContext('{"type":"session","id":"abc"}\n')
  265. expect(ctx.cwd).toBe('\0no-cwd\0')
  266. expect(ctx.cwd).not.toBe('')
  267. })
  268. it('treats an empty fixture as an empty header', () => {
  269. expect(fixtureContext('')).toEqual({ sessionIds: [], cwd: '\0no-cwd\0' })
  270. })
  271. })
  272. describe('normalizedHeaders', () => {
  273. const header = (system: string): string => JSON.stringify({
  274. type: 'request/header', seq: 0, time: 9, data: { header: { config: { model: 'm' }, system }, reason: 'initial' },
  275. })
  276. it('extracts every request/header payload in log order, normalized', () => {
  277. const id = '11111111-2222-4333-8444-555555555555'
  278. const log = `${JSON.stringify({ type: 'session', id, createdAt: 5, cwd: '/w' })}\n${header('one')}\n`
  279. + `${JSON.stringify({ type: 'turn/start', seq: 1, time: 9, data: { turn: 1 } })}\n${header('two')}\n`
  280. const headers = normalizedHeaders(log, { sessionIds: [id], cwd: '/w' })
  281. expect(headers).toEqual([
  282. { config: { model: 'm' }, system: 'one' },
  283. { config: { model: 'm' }, system: 'two' },
  284. ])
  285. })
  286. it('yields nothing for a log without header events', () => {
  287. const log = `${JSON.stringify({ type: 'session', id: 'a', createdAt: 5 })}\n`
  288. expect(normalizedHeaders(log, { sessionIds: [], cwd: '/w' })).toEqual([])
  289. })
  290. })
  291. describe('normalizedSystemPrompts', () => {
  292. it('extracts normalized string prompts and omits absent or non-string fields', () => {
  293. const log = [
  294. '{"type":"session","id":"a","createdAt":5,"cwd":"/w"}',
  295. '{"type":"request/header","seq":0,"time":9,"data":{"header":{"system":"work in /w"}}}',
  296. '{"type":"request/header","seq":1,"time":9,"data":{"header":{}}}',
  297. '{"type":"request/header","seq":2,"time":9,"data":{"header":{"system":null}}}',
  298. '{"type":"request/header","seq":3,"time":9,"data":{"header":null}}',
  299. '{"type":"request/header","seq":4,"time":9,"data":{"header":"invalid"}}',
  300. '',
  301. ].join('\n')
  302. expect(normalizedSystemPrompts(log, { sessionIds: [], cwd: '/w' })).toEqual(['work in {{cwd}}'])
  303. })
  304. })
  305. describe('normalizedToolSchemas', () => {
  306. it('extracts normalized schema arrays and omits absent or non-array fields', () => {
  307. const log = [
  308. '{"type":"session","id":"a","createdAt":5,"cwd":"/w"}',
  309. '{"type":"request/header","seq":0,"time":9,"data":{"header":{"tools":[{"name":"read","description":"work in /w"}]}}}',
  310. '{"type":"request/header","seq":1,"time":9,"data":{"header":{}}}',
  311. '{"type":"request/header","seq":2,"time":9,"data":{"header":{"tools":null}}}',
  312. '{"type":"request/header","seq":3,"time":9,"data":{"header":null}}',
  313. '{"type":"request/header","seq":4,"time":9,"data":{"header":"invalid"}}',
  314. '',
  315. ].join('\n')
  316. expect(normalizedToolSchemas(log, { sessionIds: [], cwd: '/w' })).toEqual([
  317. [{ name: 'read', description: 'work in {{cwd}}' }],
  318. ])
  319. })
  320. })
  321. describe('formatSystemPromptSnapshot', () => {
  322. it('adds a missing terminal newline without changing an existing one', () => {
  323. expect(formatSystemPromptSnapshot('prompt')).toBe('prompt\n')
  324. expect(formatSystemPromptSnapshot('prompt\n')).toBe('prompt\n')
  325. })
  326. it('renders readable changed-prompt sections', () => {
  327. expect(formatSystemPromptSnapshot('prompt', ['new\nlines']))
  328. .toBe('prompt\n\n<!-- request/header change 1 -->\n\nnew\nlines\n')
  329. })
  330. it('does not double the newline of a changed prompt', () => {
  331. expect(formatSystemPromptSnapshot('prompt\n', ['changed\n']))
  332. .toBe('prompt\n\n<!-- request/header change 1 -->\n\nchanged\n')
  333. })
  334. })
  335. describe('headerChangeCount', () => {
  336. it('counts changed request headers, ignoring anchors, blanks, and other lines', () => {
  337. const change = JSON.stringify({ type: 'request/header', seq: 2, time: 9, data: { reason: 'change' } })
  338. const anchor = JSON.stringify({ type: 'request/header', seq: 0, time: 9, data: { reason: 'initial' } })
  339. const other = JSON.stringify({ type: 'turn/start', seq: 1, time: 9, data: {} })
  340. expect(headerChangeCount(`${anchor}\n${other}\n\n${change}\n${change}\n`)).toBe(2)
  341. expect(headerChangeCount(`${anchor}\n`)).toBe(0)
  342. })
  343. })
  344. describe('tool-schema snapshots', () => {
  345. const snapshot = {
  346. initial: [{ name: 'read', description: 'Read a file.' }],
  347. changes: [[{ name: 'grep', description: 'Search files.' }]],
  348. }
  349. it('formats and parses canonical structured JSON', () => {
  350. const formatted = formatToolSchemasSnapshot(snapshot.initial, snapshot.changes)
  351. expect(formatted).toBe(`${JSON.stringify(snapshot, null, 2)}\n`)
  352. expect(parseToolSchemasSnapshot(formatted)).toEqual(snapshot)
  353. })
  354. it('rejects invalid top-level and field shapes', () => {
  355. expect(() => parseToolSchemasSnapshot('null')).toThrow(/must be an object/)
  356. expect(() => parseToolSchemasSnapshot('"invalid"')).toThrow(/must be an object/)
  357. expect(() => parseToolSchemasSnapshot('[]')).toThrow(/must be an object/)
  358. expect(() => parseToolSchemasSnapshot('{"initial":{},"changes":[]}')).toThrow(/array-valued/)
  359. expect(() => parseToolSchemasSnapshot('{"initial":[],"changes":{}}')).toThrow(/array-valued/)
  360. expect(() => parseToolSchemasSnapshot('{"initial":[],"changes":[{}]}')).toThrow(/array-valued/)
  361. })
  362. it('restores initial schemas into the pinned header token', () => {
  363. expect(restorePinnedToolSchemas({ system: '{{system}}', tools: '{{tools}}' }, snapshot.initial))
  364. .toEqual({ system: '{{system}}', tools: snapshot.initial })
  365. })
  366. it('rejects invalid headers and a missing tool token', () => {
  367. expect(() => restorePinnedToolSchemas(null, snapshot.initial)).toThrow(/must be an object/)
  368. expect(() => restorePinnedToolSchemas('invalid', snapshot.initial)).toThrow(/must be an object/)
  369. expect(() => restorePinnedToolSchemas([], snapshot.initial)).toThrow(/must be an object/)
  370. expect(() => restorePinnedToolSchemas({ tools: [] }, snapshot.initial)).toThrow(/must equal/)
  371. })
  372. })
  373. describe('unknownToolCallIds', () => {
  374. it('returns structured UNKNOWN_TOOL call ids and ignores other results', () => {
  375. const log = [
  376. '{"type":"tool/result","data":{"callId":"missing","error":{"code":"UNKNOWN_TOOL"}}}',
  377. '{"type":"tool/result","data":{"callId":"failed","error":{"code":"EXECUTION_FAILED"}}}',
  378. '{"type":"tool/result","data":null}',
  379. '{"type":"tool/result","data":"invalid"}',
  380. '{"type":"tool/result","data":{"error":null}}',
  381. '{"type":"tool/result","data":{"error":"invalid"}}',
  382. '{"type":"assistant/message","data":{"error":{"code":"UNKNOWN_TOOL"}}}',
  383. '{"type":"tool/result","data":{"error":{"code":"UNKNOWN_TOOL"}}}',
  384. '',
  385. ].join('\n')
  386. expect(unknownToolCallIds(log)).toEqual(['missing', '<missing callId>'])
  387. })
  388. it('returns no failures for ordinary tool results', () => {
  389. expect(unknownToolCallIds('{"type":"tool/result","data":{"callId":"ok"}}\n')).toEqual([])
  390. })
  391. })
  392. describe('refreshFixtureReplacements', () => {
  393. it('maps fresh ids and cwd values to the existing fixture values, skipping non-replacements', () => {
  394. const log = (content: string): HarvestedLog => ({ id: 'diagnostic', createdAt: 1, content })
  395. const logs = [
  396. log('{"type":"session","id":"","cwd":"/same"}\n'),
  397. log('{"type":"session","id":"new-parent","cwd":"/new"}\n'),
  398. log('{"type":"session","id":"new-child","cwd":"/new"}\n'),
  399. ]
  400. const fixtures = [
  401. '{"type":"session","id":"","cwd":"/same"}\n',
  402. '{"type":"session","id":"old-parent","cwd":"/old"}\n',
  403. ]
  404. expect(refreshFixtureReplacements(logs, fixtures)).toEqual([
  405. { from: 'new-parent', to: 'old-parent' },
  406. { from: '/new', to: '/old' },
  407. ])
  408. })
  409. })
  410. describe('stabilizeRefreshLog', () => {
  411. it('preserves unpacked member times when refresh first packs a chunk run', () => {
  412. const fresh = [
  413. '{"type":"session","id":"same","createdAt":200}',
  414. '{"type":"reasoning-chunks","seq0":2,"time0":200,"data":{"turn":1,"step":1,"index":0,"dt":[5,7],"texts":["new",""," split"]}}',
  415. '{"type":"assistant/message","seq":5,"time":220,"data":{}}',
  416. '',
  417. ].join('\n')
  418. const existing = [
  419. '{"type":"session","id":"same","createdAt":100}',
  420. '{"type":"assistant/chunk","seq":2,"time":100,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"old"}}}',
  421. '{"type":"assistant/chunk","seq":3,"time":101,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"chunk"}}}',
  422. '{"type":"assistant/chunk","seq":4,"time":103,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"shape"}}}',
  423. '{"type":"assistant/message","seq":5,"time":104,"data":{}}',
  424. '',
  425. ].join('\n')
  426. expect(stabilizeRefreshLog(fresh, existing, [])).toBe([
  427. '{"type":"session","id":"same","createdAt":100}',
  428. '{"type":"reasoning-chunks","seq0":2,"time0":100,"data":{"turn":1,"step":1,"index":0,"dt":[1,2],"texts":["new",""," split"]}}',
  429. '{"type":"assistant/message","seq":5,"time":104,"data":{}}',
  430. '',
  431. ].join('\n'))
  432. })
  433. it('preserves packed member times without flattening fresh chunk boundaries', () => {
  434. const fresh = [
  435. '{"type":"session","id":"same","createdAt":200}',
  436. '{"type":"text-chunks","seq0":2,"time0":200,"data":{"turn":1,"step":1,"index":0,"dt":[5,7],"texts":["new",""," split"]}}',
  437. '',
  438. ].join('\n')
  439. const existing = [
  440. '{"type":"session","id":"same","createdAt":100}',
  441. '{"type":"text-chunks","seq0":2,"time0":100,"data":{"turn":1,"step":1,"index":0,"dt":[1,2],"texts":["old","chunk","shape"]}}',
  442. '',
  443. ].join('\n')
  444. expect(stabilizeRefreshLog(fresh, existing, [])).toBe([
  445. '{"type":"session","id":"same","createdAt":100}',
  446. '{"type":"text-chunks","seq0":2,"time0":100,"data":{"turn":1,"step":1,"index":0,"dt":[1,2],"texts":["new",""," split"]}}',
  447. '',
  448. ].join('\n'))
  449. })
  450. it.each([
  451. ['the old run is absent', [], 200],
  452. ['the old run is shorter', [100, 101], 100],
  453. ['a later old time is invalid', [100, 'invalid', 103], 100],
  454. ['an old gap is not exactly representable', [Number.MIN_SAFE_INTEGER, Number.MAX_SAFE_INTEGER - 1, Number.MAX_SAFE_INTEGER - 1], Number.MIN_SAFE_INTEGER],
  455. ])('keeps fresh packed gaps when %s', (_case, existingTimes, expectedTime0) => {
  456. const freshRow = {
  457. type: 'reasoning-chunks',
  458. seq0: 2,
  459. time0: 200,
  460. data: { turn: 1, step: 1, index: 0, dt: [5, 7], texts: ['new', '', ' split'] },
  461. }
  462. const existingRows = existingTimes.map((time, index) => ({
  463. type: 'assistant/chunk',
  464. seq: index + 2,
  465. time,
  466. data: {},
  467. }))
  468. const output = stabilizeRefreshLog(
  469. `${JSON.stringify({ type: 'session', id: 'same', createdAt: 200 })}\n${JSON.stringify(freshRow)}\n`,
  470. `${JSON.stringify({ type: 'session', id: 'same', createdAt: 100 })}\n${existingRows.map(row => JSON.stringify(row)).join('\n')}\n`,
  471. [],
  472. ).trim().split('\n').map(line => JSON.parse(line) as Record<string, unknown>)
  473. expect(output[1]).toStrictEqual({ ...freshRow, time0: expectedTime0 })
  474. })
  475. it('aligns volatile times across a newly inserted log event', () => {
  476. const fresh = [
  477. '{"type":"session","id":"same","createdAt":200}',
  478. '{"type":"turn/start","seq":0,"time":21}',
  479. '{"type":"user/message","seq":1,"time":22}',
  480. '{"type":"session/title","seq":2,"time":999}',
  481. '{"type":"step/start","seq":3,"time":1000}',
  482. '{"type":"request/header","seq":4,"time":1001}',
  483. '',
  484. ].join('\n')
  485. const existing = [
  486. '{"type":"session","id":"same","createdAt":100}',
  487. '{"type":"turn/start","seq":0,"time":11}',
  488. '{"type":"user/message","seq":1,"time":12}',
  489. '{"type":"step/start","seq":2,"time":13}',
  490. '{"type":"request/header","seq":3,"time":14}',
  491. '',
  492. ].join('\n')
  493. expect(stabilizeRefreshLog(fresh, existing, [])).toBe([
  494. '{"type":"session","id":"same","createdAt":100}',
  495. '{"type":"turn/start","seq":0,"time":11}',
  496. '{"type":"user/message","seq":1,"time":12}',
  497. '{"type":"session/title","seq":2,"time":12}',
  498. '{"type":"step/start","seq":3,"time":13}',
  499. '{"type":"request/header","seq":4,"time":14}',
  500. '',
  501. ].join('\n'))
  502. })
  503. it('keeps volatile fixture fields while preserving fresh meaningful payloads', () => {
  504. const fresh = [
  505. '{"type":"session","id":"new-child","createdAt":200,"cwd":"/new","parentSession":"new-parent","seedLength":1}',
  506. '{"type":"hook/result","seq":1,"time":22,"data":{"decision":"block","durationMs":37}}',
  507. '{"type":"turn/end","seq":2,"time":33,"data":{"error":"fresh error"}}',
  508. '{"type":"tool/result","seq":3,"time":44,"data":{"text":"new-parent in /new"}}',
  509. '{"type":"hook/result","seq":4,"time":55,"data":{"decision":"allow","durationMs":5}}',
  510. '',
  511. ].join('\n')
  512. const existing = [
  513. '{"type":"session","id":"old-child","createdAt":100,"cwd":"/old","parentSession":"old-parent","seedLength":5}',
  514. '{"type":"hook/result","seq":1,"time":11,"data":{"decision":"stale","durationMs":99}}',
  515. '{"type":"turn/end","seq":2,"data":{"error":"stale"}}',
  516. '{"type":"assistant/message","seq":3,"time":12,"data":{"text":"different type"}}',
  517. '{"type":"hook/result","seq":4,"time":13,"data":{"decision":"stale"}}',
  518. '',
  519. ].join('\n')
  520. expect(stabilizeRefreshLog(fresh, existing, [
  521. { from: 'new-parent', to: 'old-parent' },
  522. { from: 'new-child', to: 'old-child' },
  523. { from: '/new', to: '/old' },
  524. ])).toBe([
  525. '{"type":"session","id":"old-child","createdAt":100,"cwd":"/old","parentSession":"old-parent","seedLength":1}',
  526. '{"type":"hook/result","seq":1,"time":11,"data":{"decision":"block","durationMs":99}}',
  527. '{"type":"turn/end","seq":2,"time":33,"data":{"error":"fresh error"}}',
  528. '{"type":"tool/result","seq":3,"time":44,"data":{"text":"old-parent in /old"}}',
  529. '{"type":"hook/result","seq":4,"time":13,"data":{"decision":"allow","durationMs":5}}',
  530. '',
  531. ].join('\n'))
  532. })
  533. })