headless.snapshot.ts 53 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199
  1. /** Recorded-session replay through the shipped headless `dsh` profile. */
  2. import { startHttpMcpFixture } from '../../packages/mcp/mcp-client/tests/http-fixture.ts'
  3. import { cp, copyFile, mkdir, mkdtemp, readFile, readdir, rm, utimes, writeFile } from 'node:fs/promises'
  4. import { existsSync } from 'node:fs'
  5. import { spawnSync } from 'node:child_process'
  6. import { tmpdir } from 'node:os'
  7. import { basename, delimiter, dirname, isAbsolute, join, relative, sep } from 'node:path'
  8. import { fileURLToPath } from 'node:url'
  9. import { describe, expect, it } from 'vitest'
  10. import ts from 'typescript'
  11. import { SESSION_FORMAT_VERSION } from '@deepseek-ai/dsh-session'
  12. import { releasedV0SessionFormatCodec } from '@deepseek-ai/dsh-session-format-v0-to-v1'
  13. import type { SessionFormatEvent, SessionFormatMigrationContext } from '@deepseek-ai/dsh-session-format'
  14. import { assertWorkspaceOutsideTemp, outsideTempWorkspaceParent } from '../../scripts/snapshot-workspace-parent.ts'
  15. import {
  16. assertPersistedSessionVersion,
  17. assertSessionFixtureVersion,
  18. captureExpectedWorkspaceSnapshot,
  19. captureWorkspaceSnapshot,
  20. fixtureContext,
  21. formatSystemPromptSnapshot,
  22. formatToolSchemasSnapshot,
  23. latestPersistedSessionPaths,
  24. materializeProfilePatch,
  25. normalizeSessionSnapshots,
  26. normalizedHeaders,
  27. normalizedSystemPrompts,
  28. normalizedToolSchemas,
  29. parseSnapshotManifest,
  30. parseToolSchemasSnapshot,
  31. redactSessionSnapshotIds,
  32. refreshFixtureReplacements,
  33. restorePinnedToolSchemas,
  34. scrubSessionSnapshot,
  35. scrubSystemPrompts,
  36. scrubToolSchemas,
  37. sessionFixtureName,
  38. systemPromptPrecedesRequests,
  39. sessionFixtureNames,
  40. sessionHeaderVersion,
  41. writerSnapshotName,
  42. snapshotSpillRoot,
  43. stabilizeFixtureMessageIds,
  44. stabilizeRefreshLog,
  45. tokenizeSessionFixtureCwd,
  46. writesCurrentSessionFixtures,
  47. type HarvestedLog,
  48. type NormalizeContext,
  49. type SnapshotManifest,
  50. type WorkspaceSnapshotEntry,
  51. } from '@deepseek-ai/dsh-session-snapshot'
  52. import { LOADER_SMOKE_TEST_TIMEOUT_MS, runLoaderSmoke } from '@deepseek-ai/dsh-loader-smoke'
  53. import { resolvePwshPath } from '@deepseek-ai/dsh-pwsh-local'
  54. import { parseSessionLog, prepareSessionSnapshotFixtureForComparison } from '@deepseek-ai/dsh-llm-replay'
  55. const repoRoot = fileURLToPath(new URL('../../', import.meta.url))
  56. const snapshotsRoot = fileURLToPath(new URL('./', import.meta.url))
  57. const dshBin = join(repoRoot, 'apps/cli/src/bin.ts')
  58. const tsconfigPath = join(repoRoot, 'tsconfig.json')
  59. const editingCordisSkill = join(
  60. repoRoot,
  61. 'packages/preset/agent-presets/presets/cordis/skills/editing-cordis-compositions/SKILL.md',
  62. )
  63. type SnapshotMode = 'replay' | 'record' | 'refresh'
  64. function snapshotMode(value: string | undefined): SnapshotMode {
  65. switch (value) {
  66. case undefined:
  67. case '':
  68. case 'replay': return 'replay'
  69. case 'record': return 'record'
  70. case 'refresh': return 'refresh'
  71. default: throw new Error(`unknown DSH_SNAPSHOT mode: ${value}`)
  72. }
  73. }
  74. const mode = snapshotMode(process.env.DSH_SNAPSHOT)
  75. const RUNTIME_WORKSPACE_ENTRIES = ['.agents', '.dsh', '.snapshot-patches'] as const
  76. interface JsonObject {
  77. [key: string]: unknown
  78. }
  79. interface HeadlessScenario {
  80. readonly name: string
  81. readonly dir: string
  82. readonly manifest: SnapshotManifest & {
  83. composition: string
  84. recording: 'live' | 'authored'
  85. header: NonNullable<SnapshotManifest['header']>
  86. }
  87. }
  88. interface SessionLog {
  89. readonly content: string
  90. readonly header: JsonObject
  91. }
  92. function propertyName(node: ts.PropertyName): string | undefined {
  93. if (ts.isIdentifier(node) || ts.isStringLiteral(node) || ts.isNumericLiteral(node)) return node.text
  94. return undefined
  95. }
  96. function bindsOsAssignedPort(argument: ts.Expression | undefined): boolean {
  97. if (argument === undefined) return false
  98. if (ts.isNumericLiteral(argument)) return Number(argument.text) === 0
  99. if (!ts.isObjectLiteralExpression(argument)) return false
  100. let portIsZero: boolean | undefined
  101. for (const property of argument.properties) {
  102. if (ts.isSpreadAssignment(property)) {
  103. portIsZero = undefined
  104. continue
  105. }
  106. if (propertyName(property.name) !== 'port') continue
  107. portIsZero = ts.isPropertyAssignment(property)
  108. && ts.isNumericLiteral(property.initializer)
  109. && Number(property.initializer.text) === 0
  110. }
  111. return portIsZero === true
  112. }
  113. function listenerPortViolations(path: string, sourceText: string): string[] {
  114. const source = ts.createSourceFile(path, sourceText, ts.ScriptTarget.Latest, true, ts.ScriptKind.JS)
  115. const violations: string[] = []
  116. const visit = (node: ts.Node): void => {
  117. if (ts.isCallExpression(node)
  118. && ts.isPropertyAccessExpression(node.expression)
  119. && node.expression.name.text === 'listen'
  120. && !bindsOsAssignedPort(node.arguments[0])) {
  121. const line = source.getLineAndCharacterOfPosition(node.getStart(source)).line + 1
  122. const received = node.arguments[0]?.getText(source) ?? '<missing>'
  123. violations.push(
  124. `${path}:${line}: listener port ${received} must use listen(0, ...) or listen({ port: 0, ... })`,
  125. )
  126. }
  127. ts.forEachChild(node, visit)
  128. }
  129. visit(source)
  130. return violations
  131. }
  132. function harvested(log: SessionLog): HarvestedLog {
  133. return {
  134. id: String(log.header.id),
  135. createdAt: Number(log.header.createdAt),
  136. ...(typeof log.header.parentSession === 'string' ? { parentSession: log.header.parentSession } : {}),
  137. content: log.content,
  138. }
  139. }
  140. function records(log: string): JsonObject[] {
  141. return log.split(/\r?\n/)
  142. .filter(line => line.trim() !== '')
  143. .map(line => JSON.parse(line) as JsonObject)
  144. }
  145. function headerOf(log: string): JsonObject {
  146. return records(log)[0] ?? {}
  147. }
  148. function contextOf(logs: readonly string[]): NormalizeContext {
  149. const headers = logs.map(headerOf)
  150. return {
  151. sessionIds: headers.flatMap(header => typeof header.id === 'string' ? [header.id] : []),
  152. cwd: typeof headers[0]?.cwd === 'string' ? headers[0].cwd : '\0missing-cwd\0',
  153. }
  154. }
  155. async function persistedSessions(cwd: string): Promise<SessionLog[]> {
  156. const root = join(cwd, '.dsh', 'sessions')
  157. const files = latestPersistedSessionPaths(await readdir(root, { recursive: true }))
  158. const logs = await Promise.all(files.map(async (file): Promise<SessionLog> => {
  159. const content = await readFile(join(root, file), 'utf8')
  160. expect(assertPersistedSessionVersion(basename(file), content), `${file}: current writer`).toBe(SESSION_FORMAT_VERSION)
  161. return { content, header: headerOf(content) }
  162. }))
  163. // Siblings bind to fixture roles by catalog publication order; concurrent
  164. // provider startup can publish an older Session after a newer one.
  165. const catalogOrders = new Map(logs.map(log => [log.header.id, new Map(
  166. records(log.content)
  167. .filter(event => event.type === 'subagent/catalog')
  168. .map((event, index) => [(event.data as JsonObject).childId, index]),
  169. )]))
  170. return logs.sort((left, right) => {
  171. const leftChild = typeof left.header.parentSession === 'string'
  172. const rightChild = typeof right.header.parentSession === 'string'
  173. if (leftChild !== rightChild) return leftChild ? 1 : -1
  174. if (left.header.parentSession === right.header.parentSession) {
  175. const catalogOrder = catalogOrders.get(left.header.parentSession)
  176. const childOrder = (catalogOrder?.get(left.header.id) ?? Infinity) - (catalogOrder?.get(right.header.id) ?? Infinity)
  177. if (childOrder) return childOrder
  178. }
  179. return Number(left.header.createdAt) - Number(right.header.createdAt)
  180. })
  181. }
  182. async function fixtureSessions(scenario: HeadlessScenario): Promise<string[]> {
  183. const files = sessionFixtureNames(await readdir(scenario.dir))
  184. return Promise.all(files.map(async (file) => {
  185. const content = await readFile(join(scenario.dir, file), 'utf8')
  186. assertSessionFixtureVersion(file, content)
  187. return content
  188. }))
  189. }
  190. async function primaryFixtureFile(dir: string): Promise<string> {
  191. const [primary] = sessionFixtureNames(await readdir(dir))
  192. if (primary === undefined) throw new Error(`${dir}: missing parent Session fixture`)
  193. const content = await readFile(join(dir, primary), 'utf8')
  194. assertSessionFixtureVersion(primary, content)
  195. return primary
  196. }
  197. async function writeSessionFixtures(
  198. scenario: HeadlessScenario,
  199. actualLogs: readonly SessionLog[],
  200. existing: readonly string[],
  201. ctx: NormalizeContext,
  202. ): Promise<string[]> {
  203. const names = actualLogs.map((log, index) => scenario.manifest.sessionFormat === undefined
  204. ? sessionFixtureName(index, sessionHeaderVersion(log.content, `harvested Session ${index}`))
  205. : writerSnapshotName(index))
  206. const prior = names.map((_, index) => existing[index] ?? '')
  207. const replacements = mode === 'refresh'
  208. ? refreshFixtureReplacements(actualLogs.map(harvested), prior)
  209. : []
  210. const fresh = actualLogs.map((log, index) => {
  211. const stable = tokenizeSessionFixtureCwd(mode === 'refresh'
  212. ? stabilizeRefreshLog(log.content, prior[index] as string, replacements, ctx)
  213. : log.content)
  214. return scrubSessionSnapshot(prepareSessionSnapshotFixtureForComparison(stable))
  215. })
  216. const output = redactSessionSnapshotIds(stabilizeFixtureMessageIds(fresh, prior))
  217. await Promise.all(output.map((content, index) => writeFile(join(scenario.dir, names[index] as string), content)))
  218. return output
  219. }
  220. /**
  221. * Write prompt and tool-schema sidecars independently of Session-generation retention.
  222. * @param scenario - scenario and sidecar ownership metadata.
  223. * @param actualLogs - current run's primary-first Session logs.
  224. * @param ctx - volatile run values used by header normalization.
  225. */
  226. async function writeHeaderSidecars(
  227. scenario: HeadlessScenario,
  228. actualLogs: readonly SessionLog[],
  229. ctx: NormalizeContext,
  230. ): Promise<void> {
  231. if (scenario.manifest.header.pin === true
  232. || [...headerPins.values()].some(pin => pin.manifest.header.systemPromptSource === scenario.name
  233. || pin.manifest.header.toolSchemasSource === scenario.name)) {
  234. const primary = actualLogs[0]
  235. if (primary === undefined) throw new Error(`${scenario.name}: write-back has no primary session`)
  236. const prompts = normalizedSystemPrompts(primary.content, ctx)
  237. const schemas = normalizedToolSchemas(primary.content, ctx)
  238. const promptOwner = scenario.manifest.header.systemPromptSource ?? scenario.name
  239. const schemaOwner = scenario.manifest.header.toolSchemasSource ?? scenario.name
  240. if (promptOwner === scenario.name) {
  241. await writeFile(
  242. join(scenario.dir, 'system-prompt.expected.md'),
  243. formatSystemPromptSnapshot(prompts[0] as string, prompts.slice(1)),
  244. )
  245. }
  246. if (schemaOwner === scenario.name) {
  247. await writeFile(
  248. join(scenario.dir, 'tool-schemas.expected.json'),
  249. formatToolSchemasSnapshot(schemas[0] as unknown[], schemas.slice(1)),
  250. )
  251. }
  252. }
  253. for (const index of scenario.manifest.header.childSystemPrompts ?? []) {
  254. const child = actualLogs[index]
  255. if (child === undefined) throw new Error(`${scenario.name}: write-back has no child ${index} prompt`)
  256. const prompts = normalizedSystemPrompts(child.content, ctx)
  257. await writeFile(
  258. join(scenario.dir, `system-prompt.${index}.expected.md`),
  259. formatSystemPromptSnapshot(prompts[0] as string, prompts.slice(1)),
  260. )
  261. }
  262. for (const index of scenario.manifest.header.childToolSchemas ?? []) {
  263. const child = actualLogs[index]
  264. if (child === undefined) throw new Error(`${scenario.name}: write-back has no child ${index} schemas`)
  265. const schemas = normalizedToolSchemas(child.content, ctx)
  266. await writeFile(
  267. join(scenario.dir, `tool-schemas.${index}.expected.json`),
  268. formatToolSchemasSnapshot(schemas[0] as unknown[], schemas.slice(1)),
  269. )
  270. }
  271. }
  272. function taskFromSession(log: string): string | undefined {
  273. const text = (value: unknown): string | undefined => {
  274. if (value === null || typeof value !== 'object') return undefined
  275. const message = value as JsonObject
  276. const source = message.source as JsonObject | undefined
  277. if (source?.kind !== 'user' || !Array.isArray(message.content)) return undefined
  278. const blocks = message.content as JsonObject[]
  279. return blocks.length === 1 && blocks[0]?.type === 'text' && typeof blocks[0].text === 'string'
  280. ? blocks[0].text
  281. : undefined
  282. }
  283. // Inbox text retains canonical mentions that pre-step renders as readable labels.
  284. for (const record of records(log)) {
  285. if (record.type !== 'agent/inbox/spliced') continue
  286. const data = record.data as JsonObject | undefined
  287. if (!Array.isArray(data?.inserted)) continue
  288. for (const message of data.inserted) {
  289. const task = text(message)
  290. if (task !== undefined) return task
  291. }
  292. }
  293. for (const record of records(log)) {
  294. if (record.type !== 'user/message') continue
  295. const task = text(record.data)
  296. if (task !== undefined) return task
  297. }
  298. return undefined
  299. }
  300. function finalTextFromSession(log: string): string {
  301. const messages = records(log).flatMap((record) => {
  302. if (record.type !== 'assistant/message') return []
  303. const data = record.data as JsonObject | undefined
  304. const message = data?.message as JsonObject | undefined
  305. return message === undefined ? [] : [message]
  306. })
  307. const content = messages.at(-1)?.content
  308. if (!Array.isArray(content)) return ''
  309. return (content as JsonObject[])
  310. .flatMap(block => block.type === 'text' && typeof block.text === 'string' ? [block.text] : [])
  311. .join('')
  312. }
  313. function turnReasonFromSession(log: string): JsonObject | undefined {
  314. const endings = records(log).flatMap((record) => {
  315. if (record.type !== 'turn/end') return []
  316. const data = record.data as JsonObject | undefined
  317. return data?.reason !== null && typeof data?.reason === 'object'
  318. ? [data.reason as JsonObject]
  319. : []
  320. })
  321. return endings.at(-1)
  322. }
  323. function stderrFromSession(log: string): string {
  324. let output = ''
  325. let started = false
  326. let open = false
  327. let endsWithNewline = true
  328. const appendReasoning = (text: string): void => {
  329. if (text === '') return
  330. if (!open) {
  331. output += 'dsh: reasoning:\n'
  332. open = true
  333. }
  334. output += text
  335. endsWithNewline = text.endsWith('\n')
  336. }
  337. const close = (): void => {
  338. if (!open) return
  339. if (!endsWithNewline) output += '\n'
  340. open = false
  341. endsWithNewline = true
  342. }
  343. const consume = (type: unknown, data: JsonObject | undefined): void => {
  344. if (type === 'reasoning-chunks') {
  345. if (!Array.isArray(data?.texts) || data.texts.some(text => typeof text !== 'string')) {
  346. throw new Error('headless snapshot reasoning chunks have invalid text')
  347. }
  348. for (const text of data.texts as string[]) appendReasoning(text)
  349. return
  350. }
  351. if (type === 'text-chunks' || type === 'tool-call-chunks') {
  352. close()
  353. return
  354. }
  355. if (type !== 'assistant/chunk' && type !== 'chunk') return
  356. const chunk = data?.chunk as JsonObject | undefined
  357. switch (chunk?.type) {
  358. case 'reasoning-delta':
  359. if (typeof chunk.text !== 'string') throw new Error('headless snapshot reasoning delta has invalid text')
  360. appendReasoning(chunk.text)
  361. break
  362. case 'block-start':
  363. if (chunk.blockType !== 'reasoning') close()
  364. break
  365. case 'block-end': {
  366. const block = chunk.block as JsonObject | undefined
  367. if (block?.type !== 'reasoning') close()
  368. break
  369. }
  370. case 'usage':
  371. break
  372. case 'text-delta':
  373. case 'tool-call-delta':
  374. case 'finish':
  375. close()
  376. break
  377. }
  378. }
  379. for (const record of records(log)) {
  380. if (record.type === 'turn/start') {
  381. close()
  382. started = true
  383. continue
  384. }
  385. if (!started) continue
  386. const data = record.data as JsonObject | undefined
  387. if ((record.type === 'assistant/message' || record.type === 'assistant/attempt')
  388. && Array.isArray(data?.stream)) {
  389. for (const entry of data.stream) {
  390. if (entry === null || typeof entry !== 'object' || Array.isArray(entry)) {
  391. throw new Error('headless snapshot embedded stream has an invalid entry')
  392. }
  393. const streamRecord = entry as JsonObject
  394. consume(streamRecord.type, streamRecord)
  395. }
  396. close()
  397. continue
  398. }
  399. consume(record.type, data)
  400. }
  401. close()
  402. const reason = turnReasonFromSession(log)
  403. if (reason?.kind !== 'error') return output
  404. const error = reason.error as JsonObject | undefined
  405. if (typeof error?.code !== 'string' || typeof error.message !== 'string') {
  406. throw new Error('headless snapshot error reason has no code and message')
  407. }
  408. return `${output}dsh: ${error.code}: ${error.message}\n`
  409. }
  410. function modelFromSession(log: string): { provider: string; model: string } {
  411. for (const record of records(log)) {
  412. if (record.type !== 'request/header') continue
  413. const data = record.data as JsonObject | undefined
  414. const header = data?.header as JsonObject | undefined
  415. const config = header?.config as JsonObject | undefined
  416. if (typeof config?.provider === 'string' && typeof config.model === 'string') {
  417. return { provider: config.provider, model: config.model }
  418. }
  419. }
  420. throw new Error('headless snapshot session has no request model')
  421. }
  422. async function seedWorkspace(scenario: HeadlessScenario, cwd: string): Promise<void> {
  423. const source = join(scenario.dir, 'workspace')
  424. if (existsSync(source)) {
  425. for (const entry of await readdir(source)) {
  426. await cp(join(source, entry), join(cwd, entry), { recursive: true, verbatimSymlinks: true })
  427. }
  428. }
  429. const setup = scenario.manifest.workspace?.setup
  430. if (setup === undefined) return
  431. const prepare = workspaceSetups[setup]
  432. if (prepare === undefined) throw new Error(`${scenario.name}: unknown workspace setup ${setup}`)
  433. await prepare(cwd)
  434. }
  435. const workspaceSetups: Record<string, (cwd: string) => Promise<void>> = {
  436. async 'office-skills'(cwd) {
  437. await cp(join(repoRoot, 'packages/skill/skill-office/assets'), join(cwd, 'office-skills'), { recursive: true })
  438. },
  439. async 'editing-cordis-skill'(cwd) {
  440. const target = join(cwd, '.dsh', 'skills', 'editing-cordis-compositions', 'SKILL.md')
  441. await mkdir(dirname(target), { recursive: true })
  442. await copyFile(editingCordisSkill, target)
  443. },
  444. async 'delimiter-path'(cwd) {
  445. const dir = join(cwd, 'scope</system-reminder>')
  446. await mkdir(dir, { recursive: true })
  447. await Promise.all([
  448. writeFile(join(dir, 'AGENTS.md'), 'Delimiter path snapshot instruction.\n'),
  449. writeFile(join(dir, 'task.txt'), 'delimiter path snapshot task\n'),
  450. ])
  451. },
  452. async 'fixed-search-mtimes'(cwd) {
  453. const tree = join(cwd, 'tree')
  454. const files = [
  455. join('archive', 'a.ts'),
  456. join('archive', 'b.ts'),
  457. join('archive', 'c.ts'),
  458. join('docs', 'guide.md'),
  459. join('src', 'index.ts'),
  460. join('test', 'spec.ts'),
  461. 'top.txt',
  462. 'notes.md',
  463. ]
  464. for (const [index, relative] of files.entries()) {
  465. const target = join(tree, relative)
  466. await mkdir(dirname(target), { recursive: true })
  467. await writeFile(target, 'fixture\n')
  468. const mtime = new Date(2000, 0, 1, 0, 0, 0, index + 1)
  469. await utimes(target, mtime, mtime)
  470. }
  471. },
  472. }
  473. async function collectScenarios(): Promise<HeadlessScenario[]> {
  474. const scenarios: HeadlessScenario[] = []
  475. for (const entry of await readdir(snapshotsRoot, { withFileTypes: true })) {
  476. if (!entry.isDirectory()) continue
  477. const dir = join(snapshotsRoot, entry.name)
  478. const manifestPath = join(dir, 'snapshot.yml')
  479. if (!existsSync(manifestPath)) continue
  480. const manifest = parseSnapshotManifest(await readFile(manifestPath, 'utf8'), manifestPath)
  481. if (manifest.profile !== 'headless' || manifest.composition === undefined) continue
  482. if (manifest.recording === undefined || manifest.header === undefined) {
  483. throw new Error(`${entry.name}: a headless corpus manifest needs recording and header metadata`)
  484. }
  485. scenarios.push({
  486. name: entry.name,
  487. dir,
  488. manifest: { ...manifest, composition: manifest.composition, recording: manifest.recording, header: manifest.header },
  489. })
  490. }
  491. return scenarios.sort((left, right) => left.name.localeCompare(right.name))
  492. }
  493. const scenarios = await collectScenarios()
  494. const hasPwsh = spawnSync(
  495. resolvePwshPath(),
  496. ['-NoLogo', '-NoProfile', '-NonInteractive', '-Command', '$true'],
  497. { encoding: 'utf8' },
  498. ).status === 0
  499. const scenarioByName = new Map(scenarios.map(scenario => [scenario.name, scenario]))
  500. const compositionOwners = new Map<string, HeadlessScenario>()
  501. const headerPins = new Map<string, HeadlessScenario>()
  502. for (const scenario of scenarios) {
  503. const { composition, header } = scenario.manifest
  504. if (existsSync(join(scenario.dir, 'cordis.yml'))) {
  505. if (compositionOwners.has(composition)) throw new Error(`headless composition ${composition} has multiple patch owners`)
  506. compositionOwners.set(composition, scenario)
  507. }
  508. if (header.pin === true) {
  509. const key = `${composition}/${header.class}`
  510. if (headerPins.has(key)) throw new Error(`headless header class ${key} has multiple pins`)
  511. headerPins.set(key, scenario)
  512. }
  513. }
  514. function ownerOf(scenario: HeadlessScenario): HeadlessScenario {
  515. const owner = compositionOwners.get(scenario.manifest.composition)
  516. if (owner === undefined) throw new Error(`${scenario.name}: composition has no cordis.yml owner`)
  517. return owner
  518. }
  519. function pinOf(scenario: HeadlessScenario): HeadlessScenario {
  520. const { composition, header } = scenario.manifest
  521. const pin = headerPins.get(`${composition}/${header.class}`)
  522. if (pin === undefined) throw new Error(`${scenario.name}: composition/header class has no pin`)
  523. return pin
  524. }
  525. /** Require successful verification and the complete canonical event before refresh can write a fixture. */
  526. async function verifySessionQuerySpill(log: string, spillRoot: string, locatorRoot: string): Promise<void> {
  527. const events = parseSessionLog(log)
  528. const results = events.flatMap(event => event.type === 'tool/result'
  529. ? event.data.message.content.filter(block => block.type === 'tool-result')
  530. : [])
  531. const readResult = results.find(result => result.toolCallId === 'call_session_query_spill')
  532. const verification = results.find(result => result.toolCallId === 'call_verify_session_query_spill')
  533. expect(readResult?.isError).toBe(false)
  534. expect(verification).toMatchObject({
  535. isError: false,
  536. content: [{ type: 'text', text: 'SPILL_CANONICAL_OK\n' }],
  537. })
  538. const preview = readResult?.content.flatMap(block => block.type === 'text' ? [block.text] : []).join('')
  539. const locator = preview?.match(/Full formatted result stored at: (.+-session_event_read\.txt)\. Use read/)
  540. expect(locator).not.toBeNull()
  541. expect(locator?.[1]).toBeDefined()
  542. expect(locator![1]!.startsWith(locatorRoot + sep)).toBe(true)
  543. const full = await readFile(join(spillRoot, relative(locatorRoot, locator![1]!)), 'utf8')
  544. const json = full.match(/```json\n([\s\S]+)\n```/)
  545. expect(json).not.toBeNull()
  546. const header = events.find(event => event.type === 'request/header')
  547. expect(header).toBeDefined()
  548. expect(JSON.parse(json![1]!)).toEqual(header)
  549. expect(full).toContain('session_event_search')
  550. }
  551. /** Require real resource results and literal instructions before recording or replay succeeds. */
  552. function verifyMcpResources(log: string, ptc: boolean): void {
  553. const events = parseSessionLog(log)
  554. const nativeResults = events.flatMap(event => event.type === 'tool/result'
  555. ? event.data.message.content.filter(block => block.type === 'tool-result')
  556. : [])
  557. const dispatches = events.flatMap(event => event.type === 'tool/ptc-dispatch' ? [event.data] : [])
  558. const results = ptc ? dispatches : nativeResults
  559. expect(results.length).toBeGreaterThanOrEqual(5)
  560. expect(results.every(result => !result.isError)).toBe(true)
  561. const calls = ptc ? dispatches.map(dispatch => dispatch.name)
  562. : events.flatMap(event => event.type === 'tool/call' ? [event.data.name] : [])
  563. expect(calls).toEqual(expect.arrayContaining(['list_mcp_resources', 'list_mcp_resource_templates', 'read_mcp_resource']))
  564. const text = results.flatMap(result => result.content
  565. .flatMap(block => block.type === 'text' ? [block.text] : [])).join('\n')
  566. expect(text).toContain('memo://text')
  567. expect(text).toContain('memo://greeting/{name}')
  568. expect(text).toContain('MCP resource text with {{braces}} intact.')
  569. expect(text).toContain('binary resource')
  570. expect(text).toContain('Hello, reader.')
  571. if (ptc) {
  572. const output = nativeResults.flatMap(result => result.content
  573. .flatMap(block => block.type === 'text' ? [block.text] : [])).join('\n')
  574. expect(output).toMatch(/"binaryAvailable"\s*:\s*true/)
  575. }
  576. expect(log).not.toContain('bWNwLXJlc291cmNlLWJpbmFyeQ==')
  577. expect(normalizedSystemPrompts(log, contextOf([log])).join('\n'))
  578. .toContain('MCP_RESOURCE_INSTRUCTION: keep {{braces}} literal.')
  579. expect(normalizedSystemPrompts(log, contextOf([log])).join('\n'))
  580. .toContain('server argument: ["catalog"]')
  581. }
  582. function verifyNoMcpServers(log: string, ptc: boolean): void {
  583. const events = parseSessionLog(log)
  584. const headers = events.flatMap(event => event.type === 'request/header' ? [event.data.header] : [])
  585. const prompts = events.flatMap(event => event.type === 'system/message'
  586. ? event.data.message.content.filter(block => block.type === 'text').map(block => block.text) : [])
  587. expect(headers.length).toBeGreaterThan(0)
  588. expect(prompts.length).toBeGreaterThan(0)
  589. for (const header of headers) {
  590. const names = header.tools?.map(tool => tool.name) ?? []
  591. expect(names.filter(name => name.includes('mcp'))).toEqual([])
  592. if (ptc) expect(names).toEqual(['run_code'])
  593. else expect(names).toContain('bash')
  594. }
  595. for (const prompt of prompts) {
  596. expect(prompt).not.toMatch(/\bMCP\b|mcp__/)
  597. expect(prompt).not.toMatch(/list_mcp_resources|list_mcp_resource_templates|read_mcp_resource/)
  598. if (ptc) expect(prompt).toContain('declare const tools:')
  599. }
  600. }
  601. /** Require an admitted failed job and zero process allocations before updating its recorded oracle. */
  602. async function verifyBackgroundConfinementFailure(log: string, cwd: string): Promise<void> {
  603. const results = parseSessionLog(log).flatMap(event => event.type === 'tool/result'
  604. ? event.data.message.content.filter(block => block.type === 'tool-result') : [])
  605. const started = results.find(result => result.toolCallId === 'async-confinement-start')
  606. const inspected = results.find(result => result.toolCallId === 'async-confinement-result')
  607. expect(started).toMatchObject({ isError: false, content: [{ type: 'text', text: 'started background job bash-1' }] })
  608. expect(inspected?.isError).toBe(false)
  609. const text = inspected?.content.flatMap(block => block.type === 'text' ? [block.text] : []).join('')
  610. expect(text).toContain('[status: failed,')
  611. expect(text).toContain('fixture asynchronous confinement refused')
  612. expect(JSON.parse(await readFile(join(cwd, 'confinement-audit.json'), 'utf8'))).toEqual({ confineCalls: 1, spawnCalls: 0 })
  613. }
  614. /** Exercise provider-cwd adoption through the shipped launcher without normalizing away the observation. */
  615. async function verifyProviderCwdResume(
  616. scenario: HeadlessScenario,
  617. cwd: string,
  618. initial: readonly SessionLog[],
  619. patches: readonly string[],
  620. model: { provider: string; model: string },
  621. fixture: string,
  622. task: string,
  623. ): Promise<void> {
  624. const providerCwd = scenario.manifest.environment?.DSH_SNAPSHOT_PROVIDER_CWD
  625. const primary = initial[0]
  626. expect(providerCwd).toBeDefined()
  627. expect(primary?.header.cwd).toBe(providerCwd)
  628. expect(primary?.header.cwd).not.toBe(cwd)
  629. const otherHostCwd = await mkdtemp(join(tmpdir(), 'dsh-provider-resume-'))
  630. const env = {
  631. DSH_HOME: join(cwd, '.dsh'),
  632. DSH_SNAPSHOT: 'replay',
  633. DSH_SNAPSHOT_FILE: fixture,
  634. DSH_SNAPSHOT_PROVIDER: model.provider,
  635. DSH_SNAPSHOT_MODEL: model.model,
  636. DSH_PERMISSION_MODE: 'read-only',
  637. DSH_TELEMETRY_DISABLED: '1',
  638. NODE_OPTIONS: [process.env.NODE_OPTIONS, '--disable-warning=ExperimentalWarning'].filter(Boolean).join(' '),
  639. }
  640. const launch = {
  641. cwd: otherHostCwd,
  642. binScript: dshBin,
  643. configPath: patches[0] as string,
  644. tsconfigPath,
  645. binArgs: [
  646. '--profile', 'headless',
  647. ...patches.flatMap(file => ['--patch', isAbsolute(file) ? file : join(cwd, file)]),
  648. '--session-id', String(primary?.header.id), task,
  649. ],
  650. }
  651. try {
  652. const refused = await runLoaderSmoke({
  653. ...launch, label: 'provider-cwd mismatched resume', expectedExitCode: 1,
  654. env: { ...env, DSH_SNAPSHOT_PROVIDER_CWD: `${providerCwd}/other` },
  655. })
  656. expect(refused.stdout).toBe('')
  657. expect(refused.stderr).toContain(`was recorded in "${providerCwd}", not "${providerCwd}/other"`)
  658. expect(await persistedSessions(cwd)).toEqual(initial)
  659. const resumed = await runLoaderSmoke({
  660. ...launch, label: 'provider-cwd matching resume',
  661. binArgs: [...launch.binArgs.slice(0, -1), '--json', task],
  662. env: { ...env, DSH_SNAPSHOT_PROVIDER_CWD: providerCwd },
  663. })
  664. const output = records(resumed.stdout)
  665. expect(output[0]).toEqual({ type: 'session', sessionId: primary?.header.id, cwd: providerCwd })
  666. expect(output.at(-1)).toEqual({ type: 'final', text: finalTextFromSession(primary!.content) })
  667. const continued = await persistedSessions(cwd)
  668. expect(continued).toHaveLength(1)
  669. expect(continued[0]?.header).toEqual(primary?.header)
  670. expect(continued[0]?.content.startsWith(primary!.content)).toBe(true)
  671. expect(records(continued[0]!.content).filter(event => event.type === 'turn/start')).toHaveLength(2)
  672. } finally {
  673. await rm(otherHostCwd, { recursive: true, force: true })
  674. }
  675. }
  676. async function verifyHeaders(scenario: HeadlessScenario, actualLogs: readonly SessionLog[], ctx: NormalizeContext): Promise<void> {
  677. const pin = pinOf(scenario)
  678. const fixture = await readFile(join(pin.dir, await primaryFixtureFile(pin.dir)), 'utf8')
  679. const pinned = normalizedHeaders(fixture, fixtureContext(fixture))
  680. const changes = pin.manifest.header.changes ?? 0
  681. expect(pinned, `${scenario.name}: pin header count`).toHaveLength(1 + changes)
  682. const promptOwner = scenarioByName.get(pin.manifest.header.systemPromptSource ?? pin.name)
  683. const schemaOwner = scenarioByName.get(pin.manifest.header.toolSchemasSource ?? pin.name)
  684. if (promptOwner === undefined || schemaOwner === undefined) {
  685. throw new Error(`${scenario.name}: header sidecar source is not a headless scenario`)
  686. }
  687. const prompt = await readFile(join(promptOwner.dir, 'system-prompt.expected.md'), 'utf8')
  688. const schemas = parseToolSchemasSnapshot(await readFile(join(schemaOwner.dir, 'tool-schemas.expected.json'), 'utf8'))
  689. const schemaSets = [schemas.initial, ...schemas.changes]
  690. expect(schemaSets, `${scenario.name}: pin tool-schema count`).toHaveLength(pinned.length)
  691. const reconstructed = pinned.map((header, index) => restorePinnedToolSchemas(
  692. header,
  693. schemaSets[index] as unknown[],
  694. ))
  695. const childPrompts = new Map<number, string>()
  696. const childSchemas = new Map<number, unknown[][]>()
  697. for (const index of scenario.manifest.header.childSystemPrompts ?? []) {
  698. childPrompts.set(index, await readFile(join(scenario.dir, `system-prompt.${index}.expected.md`), 'utf8'))
  699. }
  700. for (const index of scenario.manifest.header.childToolSchemas ?? []) {
  701. const child = parseToolSchemasSnapshot(await readFile(join(scenario.dir, `tool-schemas.${index}.expected.json`), 'utf8'))
  702. childSchemas.set(index, [child.initial, ...child.changes])
  703. }
  704. for (const [logIndex, log] of actualLogs.entries()) {
  705. const headers = normalizedHeaders(log.content, ctx)
  706. const prompts = normalizedSystemPrompts(log.content, ctx)
  707. if (headers.length > 0) {
  708. expect(systemPromptPrecedesRequests(log.content), `${scenario.name}: a system/message precedes the first request/header`).toBe(true)
  709. expect(prompts.length, `${scenario.name}: system/message count`)
  710. .toBe(1 + (logIndex === 0 ? pin.manifest.header.promptChanges ?? 0 : 0))
  711. }
  712. for (const [index, header] of headers.entries()) {
  713. const selectedSchemas = childSchemas.get(logIndex)?.[index]
  714. const base = reconstructed[index] ?? reconstructed[0]
  715. const expected = selectedSchemas === undefined ? base : { ...base as JsonObject, tools: selectedSchemas }
  716. expect(header, `${scenario.name}: request header ${index + 1}`).toEqual(expected)
  717. }
  718. if (prompts.length > 0) {
  719. expect(
  720. formatSystemPromptSnapshot(prompts[0] as string, prompts.slice(1)),
  721. `${scenario.name}: system prompts`,
  722. ).toBe(childPrompts.get(logIndex) ?? prompt)
  723. }
  724. }
  725. }
  726. describe('headless recorded-session snapshots', () => {
  727. it('gives every composition and header class exactly one current-writer pin', () => {
  728. for (const scenario of scenarios) {
  729. expect(ownerOf(scenario), `${scenario.name}: composition owner`).toBeDefined()
  730. expect(pinOf(scenario).manifest.sessionFormat, `${scenario.name}: current-writer header pin`).toBeUndefined()
  731. }
  732. })
  733. it('recognizes the supported OS-assigned listener forms', () => {
  734. expect(listenerPortViolations('accepted.mjs', [
  735. "server.listen(0, '127.0.0.1')",
  736. "server.listen({ port: 0, host: '127.0.0.1' })",
  737. 'server.listen({ ...options, port: 0 })',
  738. ].join('\n'))).toEqual([])
  739. expect(listenerPortViolations('fixed.mjs', 'server.listen(43118)')).toEqual([
  740. 'fixed.mjs:1: listener port 43118 must use listen(0, ...) or listen({ port: 0, ... })',
  741. ])
  742. expect(listenerPortViolations('dynamic.mjs', 'server.listen({ port, ...options })')).toEqual([
  743. 'dynamic.mjs:1: listener port { port, ...options } must use listen(0, ...) or listen({ port: 0, ... })',
  744. ])
  745. })
  746. it('binds scenario HTTP fixtures only to OS-assigned ports', async () => {
  747. const fixtureNames = (await readdir(snapshotsRoot, { recursive: true })).filter(name => name.endsWith('.mjs'))
  748. const violations = (await Promise.all(fixtureNames.map(async (fixtureName) => listenerPortViolations(
  749. fixtureName,
  750. await readFile(join(snapshotsRoot, fixtureName), 'utf8'),
  751. )))).flat()
  752. expect(violations).toEqual([])
  753. })
  754. it('stores session-owned inputs with typed redaction and no ACP transcript', async () => {
  755. for (const scenario of scenarios) {
  756. const fixtures = await fixtureSessions(scenario)
  757. expect(redactSessionSnapshotIds(fixtures), `${scenario.name}: identity redaction fixed point`).toEqual(fixtures)
  758. for (const fixture of fixtures) {
  759. expect(scrubSystemPrompts(fixture), `${scenario.name}: system prompt stays in a sidecar`).toBe(fixture)
  760. expect(scrubToolSchemas(fixture), `${scenario.name}: tool schemas stay in a sidecar`).toBe(fixture)
  761. }
  762. expect(existsSync(join(scenario.dir, 'input.json')), `${scenario.name}: task comes from session JSONL`).toBe(false)
  763. expect(existsSync(join(scenario.dir, 'stdout.expected.jsonl')), `${scenario.name}: no ACP transcript`).toBe(false)
  764. }
  765. })
  766. it('keeps packed chunk rows logically equal to their unpacked recording', async () => {
  767. const packedDir = join(snapshotsRoot, 'packed-chunks')
  768. const packed = await readFile(join(packedDir, await primaryFixtureFile(packedDir)), 'utf8')
  769. const [header, ...rows] = records(packed)
  770. const packedTypes = new Set(['text-chunks', 'reasoning-chunks', 'tool-call-chunks'])
  771. expect([...new Set(rows.filter(row => packedTypes.has(String(row.type))).map(row => row.type))].sort())
  772. .toStrictEqual(['reasoning-chunks', 'text-chunks', 'tool-call-chunks'])
  773. const decoder = releasedV0SessionFormatCodec.createDecoder({ ...header, cwd: '/snapshot' }, 'strict')
  774. const expanded: SessionFormatEvent[] = []
  775. const output: SessionFormatMigrationContext = {
  776. emitEvent: event => { expanded.push(event) },
  777. emitRun: run => { expanded.push(...run.expand()) },
  778. }
  779. let seq = 0
  780. for (const row of rows) {
  781. if (packedTypes.has(String(row.type))) {
  782. const data = row.data as JsonObject
  783. const values = (row.type === 'tool-call-chunks' ? data.args : data.texts) as unknown[]
  784. decoder.decodeRow({ ...row, seq0: seq, time0: 0 }, output)
  785. seq += values.length
  786. } else {
  787. decoder.decodeRow({ ...row, seq: seq++, time: 0 }, output)
  788. }
  789. }
  790. decoder.finish(output)
  791. const unpacked = [header, ...expanded.map(({ seq: _seq, time: _time, ...event }) => event)]
  792. .map(row => JSON.stringify(row)).join('\n') + '\n'
  793. const context = contextOf([packed])
  794. expect(normalizeSessionSnapshots([packed], context)).toEqual(normalizeSessionSnapshots([unpacked], context))
  795. })
  796. it('replays original inbox mentions before normalized user messages', () => {
  797. const message = (text: string) => ({ source: { kind: 'user' }, content: [{ type: 'text', text }] })
  798. const original = 'Use @[Research](dsh-session:InJlZmVyZW5jZS1zb3VyY2Ui)'
  799. const log = [
  800. { type: 'agent/inbox/spliced', data: { inserted: [message(original)] } },
  801. { type: 'user/message', data: message('Use @Research') },
  802. ].map(record => JSON.stringify(record)).join('\n')
  803. expect(taskFromSession(log)).toBe(original)
  804. expect(taskFromSession(JSON.stringify({ type: 'user/message', data: message('legacy task') }))).toBe('legacy task')
  805. })
  806. it('reconstructs reasoning stderr across packed output boundaries', () => {
  807. const log = [
  808. { type: 'turn/start', data: { turn: 1 } },
  809. { type: 'reasoning-chunks', data: { texts: ['first', ''] } },
  810. { type: 'text-chunks', data: { texts: ['text'] } },
  811. { type: 'reasoning-chunks', data: { texts: ['second'] } },
  812. { type: 'tool-call-chunks', data: { args: ['{}'] } },
  813. { type: 'reasoning-chunks', data: { texts: ['third\n'] } },
  814. { type: 'turn/end', data: { turn: 1, reason: { kind: 'completed' } } },
  815. ].map(record => JSON.stringify(record)).join('\n')
  816. expect(stderrFromSession(log)).toBe([
  817. 'dsh: reasoning:',
  818. 'first',
  819. 'dsh: reasoning:',
  820. 'second',
  821. 'dsh: reasoning:',
  822. 'third',
  823. '',
  824. ].join('\n'))
  825. })
  826. it.each(['assistant/message', 'assistant/attempt'] as const)(
  827. 'reconstructs reasoning stderr from embedded current %s streams',
  828. (eventType) => {
  829. const log = [
  830. { type: 'turn/start', data: { turn: 1 } },
  831. {
  832. type: eventType,
  833. data: {
  834. stream: [
  835. { type: 'chunk', chunk: { type: 'block-start', index: 0, blockType: 'reasoning' } },
  836. { type: 'reasoning-chunks', texts: ['first', ' thought'] },
  837. { type: 'chunk', chunk: { type: 'block-start', index: 1, blockType: 'text' } },
  838. ],
  839. },
  840. },
  841. { type: 'turn/end', data: { turn: 1, reason: { kind: 'completed' } } },
  842. ].map(record => JSON.stringify(record)).join('\n')
  843. expect(stderrFromSession(log)).toBe('dsh: reasoning:\nfirst thought\n')
  844. },
  845. )
  846. it('closes reasoning between embedded Assistant settlements', () => {
  847. const log = [
  848. { type: 'turn/start', data: { turn: 1 } },
  849. {
  850. type: 'assistant/attempt',
  851. data: { stream: [{ type: 'reasoning-chunks', texts: ['first', ' thought'] }] },
  852. },
  853. {
  854. type: 'assistant/message',
  855. data: {
  856. stream: [
  857. { type: 'chunk', chunk: { type: 'reasoning-delta', index: 0, text: 'second' } },
  858. { type: 'chunk', chunk: { type: 'finish', reason: { kind: 'stop' } } },
  859. ],
  860. },
  861. },
  862. { type: 'turn/end', data: { turn: 1, reason: { kind: 'completed' } } },
  863. ].map(record => JSON.stringify(record)).join('\n')
  864. expect(stderrFromSession(log)).toBe('dsh: reasoning:\nfirst thought\ndsh: reasoning:\nsecond\n')
  865. })
  866. it.each([10, 20])('assigns sibling roles by catalog order when the first child timestamp is %i', async (firstCreatedAt) => {
  867. const cwd = await mkdtemp(join(tmpdir(), 'dsh-headless-catalog-order-'))
  868. try {
  869. const logs = [
  870. [
  871. { type: 'session', version: SESSION_FORMAT_VERSION, id: 'parent', createdAt: 1 },
  872. { type: 'subagent/catalog', data: { childId: 'child-z' } },
  873. { type: 'subagent/catalog', data: { childId: 'child-a' } },
  874. ],
  875. [{ type: 'session', version: SESSION_FORMAT_VERSION, id: 'child-z', createdAt: firstCreatedAt, parentSession: 'parent' }],
  876. [{ type: 'session', version: SESSION_FORMAT_VERSION, id: 'child-a', createdAt: 10, parentSession: 'parent' }],
  877. ].map(rows => rows.map(row => JSON.stringify(row)).join('\n') + '\n')
  878. for (const content of logs) {
  879. const directory = join(cwd, '.dsh', 'sessions', String(headerOf(content).id))
  880. await mkdir(directory, { recursive: true })
  881. await writeFile(join(directory, `session.v${SESSION_FORMAT_VERSION}.jsonl`), content)
  882. }
  883. const actual = await persistedSessions(cwd)
  884. expect(actual.map(log => log.header.id)).toEqual(['parent', 'child-z', 'child-a'])
  885. expect(actual.map(log => log.content)).toEqual(logs)
  886. } finally {
  887. await rm(cwd, { recursive: true, force: true })
  888. }
  889. })
  890. it('writes header sidecars without replacing a retained Session generation', async () => {
  891. const directory = await mkdtemp(join(tmpdir(), 'dsh-headless-sidecars-'))
  892. try {
  893. const scenario: HeadlessScenario = {
  894. name: 'retained-pin',
  895. dir: directory,
  896. manifest: {
  897. version: 1,
  898. scenario: 'retained-pin',
  899. profile: 'headless',
  900. composition: 'default',
  901. recording: 'live',
  902. header: { class: 'default', pin: true },
  903. sessionFormat: { version: 1, coverage: ['adjacent-migration'] },
  904. },
  905. }
  906. const header = {
  907. type: 'session', version: SESSION_FORMAT_VERSION, id: 'sidecar-session', createdAt: 1,
  908. cwd: '/tmp/sidecar-session', isSeeded: false, delegationDepth: 0,
  909. }
  910. const content = [
  911. header,
  912. { type: 'turn/start', seq: 0, time: 2, data: { turn: 1 } },
  913. { type: 'step/start', seq: 1, time: 2, data: { turn: 1, step: 1 } },
  914. { type: 'system/message', seq: 2, time: 3, data: {
  915. turn: 1, step: 1,
  916. message: { role: 'system', content: [{ type: 'text', text: 'fresh system prompt' }],
  917. source: { kind: 'plugin', plugin: '@deepseek-ai/dsh-system-prompt' }, id: 'fresh-msg' },
  918. }, surfaceOp: 'append' },
  919. {
  920. type: 'request/header',
  921. seq: 3,
  922. time: 4,
  923. data: {
  924. header: {
  925. config: { provider: 'test', model: 'test' },
  926. tools: [{ name: 'fresh_tool', description: 'fresh schema', parameters: {} }],
  927. },
  928. reason: 'initial',
  929. },
  930. },
  931. ].map(record => JSON.stringify(record)).join('\n')
  932. const retained = '{"type":"session","version":1}\n'
  933. await writeFile(join(directory, 'session.v1.jsonl'), retained)
  934. await writeHeaderSidecars(
  935. scenario,
  936. [{ content, header }],
  937. { sessionIds: ['sidecar-session'], cwd: '/tmp/sidecar-session' },
  938. )
  939. expect(await readFile(join(directory, 'system-prompt.expected.md'), 'utf8'))
  940. .toBe('fresh system prompt\n')
  941. expect(await readFile(join(directory, 'tool-schemas.expected.json'), 'utf8'))
  942. .toContain('"name": "fresh_tool"')
  943. expect(await readFile(join(directory, 'session.v1.jsonl'), 'utf8')).toBe(retained)
  944. expect(sessionFixtureNames(await readdir(directory))).toEqual(['session.v1.jsonl'])
  945. } finally {
  946. await rm(directory, { recursive: true, force: true })
  947. }
  948. })
  949. for (const scenario of scenarios) {
  950. const skipped = scenario.manifest.platform === 'posix' && process.platform === 'win32'
  951. || scenario.manifest.platform === 'pwsh' && !hasPwsh
  952. || mode === 'record' && scenario.manifest.recording === 'authored'
  953. || mode === 'record' && scenario.manifest.sessionFormat !== undefined
  954. const scenarioTest = skipped ? it.skip : mode === 'replay' ? it.concurrent : it
  955. scenarioTest(`${mode}s ${scenario.name} through dsh --profile headless`, async () => {
  956. let fixtures = await fixtureSessions(scenario)
  957. const primaryFixture = fixtures[0]
  958. if (primaryFixture === undefined) throw new Error(`${scenario.name}: missing primary session fixture`)
  959. const task = taskFromSession(primaryFixture) ?? scenario.manifest.input?.task
  960. if (task === undefined) throw new Error(`${scenario.name}: no accepted or exceptional task input`)
  961. const pin = pinOf(scenario)
  962. let model: { provider: string; model: string }
  963. try {
  964. model = modelFromSession(primaryFixture)
  965. } catch {
  966. model = modelFromSession(await readFile(join(pin.dir, await primaryFixtureFile(pin.dir)), 'utf8'))
  967. }
  968. const composition = ownerOf(scenario)
  969. const baseComposition = compositionOwners.get('default')
  970. if (baseComposition === undefined) throw new Error('headless corpus has no default composition')
  971. let fixtureFiles = sessionFixtureNames(await readdir(scenario.dir))
  972. const replaying = mode !== 'record'
  973. const compositionPatch = join(composition.dir, replaying ? 'cordis.snapshot.yml' : 'cordis.yml')
  974. const patchSources = [
  975. join(baseComposition.dir, 'cordis.yml'),
  976. ...composition === baseComposition && !replaying ? [] : [compositionPatch],
  977. join(baseComposition.dir, 'model.cordis.yml'),
  978. ]
  979. const patchRoot = '.snapshot-patches'
  980. const patches = patchSources.map((source, index) => source.endsWith('.snapshot.yml')
  981. ? join(patchRoot, `${String(index)}-${basename(source)}`)
  982. : source)
  983. let actualLogs: SessionLog[] = []
  984. let initialWorkspace: WorkspaceSnapshotEntry[] | undefined
  985. let finalWorkspace: WorkspaceSnapshotEntry[] | undefined
  986. const spillRoot = await mkdtemp(join(tmpdir(), 'acp-snap-spill-'))
  987. const locatorRoot = snapshotSpillRoot(join(scenario.dir, fixtureFiles[0] as string))
  988. const mcpDemo = scenario.name === 'plugin-manager-mcp' ? await startHttpMcpFixture() : undefined
  989. let result: Awaited<ReturnType<typeof runLoaderSmoke>>
  990. try {
  991. result = await runLoaderSmoke({
  992. label: `${scenario.name} headless snapshot`,
  993. tempDirPrefix: 'dsh-log-snap-',
  994. ...(scenario.manifest.workspace?.parent === 'outside-temp' ? { tempDirParent: outsideTempWorkspaceParent() } : {}),
  995. binScript: dshBin,
  996. configPath: join(baseComposition.dir, 'cordis.yml'),
  997. binArgs: [
  998. '--profile', 'headless',
  999. ...patches.flatMap(file => ['--patch', file]),
  1000. task,
  1001. ],
  1002. tsconfigPath,
  1003. expectedExitCode: turnReasonFromSession(primaryFixture)?.kind === 'completed'
  1004. || turnReasonFromSession(primaryFixture) === undefined && scenario.manifest.input?.task !== undefined
  1005. ? 0
  1006. : 1,
  1007. env: {
  1008. DSH_SNAPSHOT: replaying ? 'replay' : 'record',
  1009. DSH_SNAPSHOT_PROVIDER: model.provider,
  1010. DSH_SNAPSHOT_MODEL: model.model,
  1011. DSH_SNAPSHOT_SPILL_ROOT: spillRoot,
  1012. DSH_SNAPSHOT_SPILL_LOCATOR_ROOT: locatorRoot,
  1013. DSH_SNAPSHOT_FILE: join(scenario.dir, fixtureFiles[0] as string),
  1014. ...(replaying && fixtureFiles.length > 1
  1015. ? { DSH_SNAPSHOT_CHILD_FILES: fixtureFiles.slice(1).map(file => join(scenario.dir, file)).join(delimiter) }
  1016. : {}),
  1017. ...(replaying && scenario.manifest.replay?.override === true
  1018. ? { DSH_SNAPSHOT_OVERRIDE: join(scenario.dir, 'replay.override.json') }
  1019. : {}),
  1020. ...(scenario.manifest.permission === undefined
  1021. ? {}
  1022. : { DSH_PERMISSION_MODE: scenario.manifest.permission }),
  1023. ...scenario.manifest.environment,
  1024. ...(scenario.name === 'mcp-resources' || scenario.name === 'mcp-resources-ptc' ? {
  1025. DSH_MCP_RESOURCES_FIXTURE: join(repoRoot, 'packages/mcp/mcp-client/tests/fixtures/resources-server.ts'),
  1026. } : {}),
  1027. NODE_OPTIONS: [process.env.NODE_OPTIONS, '--disable-warning=ExperimentalWarning'].filter(Boolean).join(' '),
  1028. DSH_TELEMETRY_DISABLED: '1',
  1029. ...(mcpDemo === undefined ? {} : { DSH_MCP_DEMO_URL: mcpDemo.url }),
  1030. },
  1031. prepare: async (cwd) => {
  1032. if (scenario.manifest.workspace?.parent === 'outside-temp') assertWorkspaceOutsideTemp(cwd)
  1033. await mkdir(join(cwd, patchRoot), { recursive: true })
  1034. patchSources.forEach((source, index) => {
  1035. if (source.endsWith('.snapshot.yml')) {
  1036. materializeProfilePatch(source, cwd, 'headless', join(cwd, patchRoot), index)
  1037. }
  1038. })
  1039. if (mcpDemo !== undefined) {
  1040. const profileDir = join(cwd, '.dsh/profiles/headless')
  1041. await mkdir(profileDir, { recursive: true })
  1042. await copyFile(join(scenario.dir, 'profile.patch.yml'), join(profileDir, 'cordis.patch.yml'))
  1043. }
  1044. await seedWorkspace(scenario, cwd)
  1045. initialWorkspace = await captureWorkspaceSnapshot(cwd, {
  1046. ignoredRootEntries: RUNTIME_WORKSPACE_ENTRIES,
  1047. })
  1048. },
  1049. inspect: async (cwd) => {
  1050. actualLogs = await persistedSessions(cwd)
  1051. if (mcpDemo !== undefined) {
  1052. const log = actualLogs[0]!.content
  1053. expect(log).toContain('mcp__demo__ping')
  1054. expect(log).toContain('pong')
  1055. expect(mcpDemo.calls).toEqual(['ping'])
  1056. const saved = await readFile(join(cwd, '.dsh/profiles/headless/cordis.patch.yml'), 'utf8')
  1057. expect(saved).toContain('id: demo')
  1058. expect(saved).toContain('disabled: false')
  1059. }
  1060. if (scenario.name === 'session-query-spill') {
  1061. await verifySessionQuerySpill(actualLogs[0]!.content, spillRoot, locatorRoot)
  1062. }
  1063. if (scenario.name === 'mcp-resources' || scenario.name === 'mcp-resources-ptc') {
  1064. verifyMcpResources(actualLogs[0]!.content, scenario.name === 'mcp-resources-ptc')
  1065. }
  1066. if (scenario.name === 'mcp-empty' || scenario.name === 'mcp-empty-ptc') {
  1067. verifyNoMcpServers(actualLogs[0]!.content, scenario.name === 'mcp-empty-ptc')
  1068. }
  1069. if (scenario.name === 'provider-cwd') {
  1070. await verifyProviderCwdResume(
  1071. scenario, cwd, actualLogs, patches, model, join(scenario.dir, fixtureFiles[0] as string), task,
  1072. )
  1073. }
  1074. if (scenario.name === 'background-confinement-failure') {
  1075. await verifyBackgroundConfinementFailure(actualLogs[0]!.content, cwd)
  1076. }
  1077. finalWorkspace = await captureWorkspaceSnapshot(cwd, {
  1078. ignoredRootEntries: RUNTIME_WORKSPACE_ENTRIES,
  1079. })
  1080. },
  1081. })
  1082. } finally {
  1083. await mcpDemo?.close()
  1084. await rm(spillRoot, { recursive: true, force: true })
  1085. }
  1086. const stderrLog = mode === 'replay' ? primaryFixture : actualLogs[0]?.content
  1087. if (stderrLog === undefined) throw new Error(`${scenario.name}: stderr projection has no primary session`)
  1088. const expectedStderr = stderrFromSession(stderrLog)
  1089. const actualContext = contextOf(actualLogs.map(log => log.content))
  1090. if (writesCurrentSessionFixtures(scenario.manifest, mode)) {
  1091. fixtures = await writeSessionFixtures(
  1092. scenario,
  1093. actualLogs,
  1094. fixtures,
  1095. actualContext,
  1096. )
  1097. fixtureFiles = actualLogs.map((log, index) => sessionFixtureName(
  1098. index,
  1099. sessionHeaderVersion(log.content, `harvested Session ${index}`),
  1100. ))
  1101. }
  1102. if (mode !== 'replay') await writeHeaderSidecars(scenario, actualLogs, actualContext)
  1103. expect(result.stdout).toBe(`${finalTextFromSession(fixtures[0] as string)}\n`)
  1104. expect(result.stderr).toBe(expectedStderr)
  1105. expect(actualLogs, `${scenario.name}: persisted session count`).toHaveLength(fixtures.length)
  1106. const writerFiles = (await readdir(scenario.dir)).filter(name => /^writer(?:\.[1-9]\d*)?\.expected\.jsonl$/u.test(name)).sort()
  1107. if (mode === 'replay') {
  1108. expect(writerFiles, 'native writer oracle inventory').toEqual(scenario.manifest.sessionFormat === undefined
  1109. ? [] : fixtures.map((_, index) => writerSnapshotName(index)).sort())
  1110. }
  1111. let expected = fixtures
  1112. if (scenario.manifest.sessionFormat !== undefined) {
  1113. if (mode === 'refresh') await writeSessionFixtures(scenario, actualLogs, fixtures, actualContext)
  1114. expected = await Promise.all(fixtures.map((_, index) => readFile(join(scenario.dir, writerSnapshotName(index)), 'utf8')))
  1115. for (const [index, content] of expected.entries()) {
  1116. expect(sessionHeaderVersion(content, writerSnapshotName(index))).toBe(SESSION_FORMAT_VERSION)
  1117. }
  1118. expect(await fixtureSessions(scenario), 'historical replay input remains unchanged').toEqual(fixtures)
  1119. }
  1120. const actualSnapshots = normalizeSessionSnapshots(actualLogs.map(log => log.content), actualContext)
  1121. const expectedSnapshots = normalizeSessionSnapshots(expected, contextOf(expected))
  1122. for (const [index, actual] of actualSnapshots.entries()) {
  1123. expect(records(actual), `${scenario.name}: session ${index}`).toEqual(records(expectedSnapshots[index] as string))
  1124. }
  1125. await verifyHeaders(scenario, actualLogs, actualContext)
  1126. if (initialWorkspace === undefined || finalWorkspace === undefined) {
  1127. throw new Error(`${scenario.name}: workspace was not captured around the profile run`)
  1128. }
  1129. if (scenario.manifest.workspace?.final === true) {
  1130. const expectedWorkspace = await captureExpectedWorkspaceSnapshot(join(scenario.dir, 'workspace.expected'))
  1131. expect(finalWorkspace, `${scenario.name}: complete final workspace`).toEqual(expectedWorkspace)
  1132. } else {
  1133. expect(finalWorkspace, `${scenario.name}: a changed workspace requires workspace.final`).toEqual(initialWorkspace)
  1134. }
  1135. }, scenario.name === 'provider-cwd' ? 3 * LOADER_SMOKE_TEST_TIMEOUT_MS : LOADER_SMOKE_TEST_TIMEOUT_MS)
  1136. }
  1137. })