resume.spec.ts 62 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334
  1. import { ToolCallId, createMessage, createUserMessage } from '@deepseek-ai/dsh-llm'
  2. import { afterEach, describe, expect, it, vi, type MockInstance } from 'vitest'
  3. import { Context } from '@deepseek-ai/cordis'
  4. import { appendFile, mkdtemp, readdir, rm } from 'node:fs/promises'
  5. import { tmpdir } from 'node:os'
  6. import { join } from 'node:path'
  7. import LlmRuntime from '@deepseek-ai/dsh-llm'
  8. import SessionStore, { SessionLogOffset, SessionSeq, Session, SessionId, TOOL_OUTCOME_UNKNOWN } from '@deepseek-ai/dsh-session'
  9. import type { SessionEvent } from '@deepseek-ai/dsh-session'
  10. import SystemPrompt from '@deepseek-ai/dsh-system-prompt'
  11. import ToolRuntime from '@deepseek-ai/dsh-tools'
  12. import AgentRegistry, { type Agent } from '@deepseek-ai/dsh-agent'
  13. import type { SessionHandle } from '@deepseek-ai/dsh-session-persistence'
  14. import JsonlSessionPersistence from '@deepseek-ai/dsh-session-persistence-jsonl'
  15. import AgentLoop from '@deepseek-ai/dsh-agent-loop'
  16. import SessionProjectionRegistry from '@deepseek-ai/dsh-session-projection'
  17. import { MockAdapter, textResponse } from './mock-adapter.ts'
  18. const dirs: string[] = []
  19. afterEach(async () => { for (const d of dirs.splice(0)) await rm(d, { recursive: true, force: true }) })
  20. async function persistentHarness(adapter: MockAdapter): Promise<{ ctx: Context; root: string }> {
  21. const root = await mkdtemp(join(tmpdir(), 'dsh-resume-'))
  22. dirs.push(root)
  23. return { ctx: await mountPersistentHarness(root, adapter), root }
  24. }
  25. async function mountPersistentHarness(root: string, adapter: MockAdapter, compression?: 'none'): Promise<Context> {
  26. const ctx = new Context()
  27. await ctx.plugin(LlmRuntime)
  28. await ctx.plugin(SessionStore)
  29. await ctx.plugin(SessionProjectionRegistry)
  30. await ctx.plugin(SystemPrompt)
  31. await ctx.plugin(ToolRuntime)
  32. await ctx.plugin(AgentRegistry)
  33. // The backend mounts BEFORE the loop so root teardown unwinds the loop
  34. // first: live agents drain their writers into still-open handles.
  35. await ctx.plugin(JsonlSessionPersistence, { root, ...compression === undefined ? {} : { compression } })
  36. await ctx.plugin(AgentLoop, { agents: [] })
  37. ctx.llm.registerAdapter(['mock'], adapter)
  38. return ctx
  39. }
  40. /** Remove every `session.lock` under the root: the POSIX forfeit-by-unlink escape hatch, without importing backend internals. */
  41. async function removeSessionLocks(dir: string): Promise<void> {
  42. for (const entry of await readdir(dir, { withFileTypes: true })) {
  43. const path = join(dir, entry.name)
  44. if (entry.isDirectory()) await removeSessionLocks(path)
  45. else if (entry.name === 'session.lock') await rm(path, { force: true })
  46. }
  47. }
  48. /** Seed one stored session through the persistence seam (header minted by the store). */
  49. async function seedStoredSession(ctx: Context, sessionId: SessionId, events: readonly SessionEvent[]): Promise<void> {
  50. const detached = ctx.sessions.prepare(sessionId)
  51. const handle = await ctx.sessionPersistence.create(detached.header)
  52. await handle.append(events)
  53. await handle.close()
  54. }
  55. /** Read one stored session's physical validated log through a read handle. */
  56. async function readStoredEvents(ctx: Context, sessionId: SessionId): Promise<readonly SessionEvent[]> {
  57. const handle = await ctx.sessionPersistence.open(sessionId, 'read')
  58. try {
  59. return (await handle.read()).events
  60. } finally {
  61. await handle.close()
  62. }
  63. }
  64. async function persistSession(sessionId: SessionId): Promise<string> {
  65. const { ctx, root } = await persistentHarness(new MockAdapter([]))
  66. // Persistence deliberately has no artifact for a truly empty session. A
  67. // balanced completed turn is the smallest resumable log and avoids running
  68. // the model merely to construct this lifecycle fixture.
  69. const seed: SessionEvent[] = [
  70. { type: 'turn/start', seq: SessionSeq(0), time: 1, data: { turn: 1 } },
  71. { type: 'turn/end', seq: SessionSeq(1), time: 2, data: { turn: 1, reason: { kind: 'completed' } } },
  72. ]
  73. await seedStoredSession(ctx, sessionId, seed)
  74. await ctx.fiber.dispose()
  75. return root
  76. }
  77. /** A handle stand-in for abandoned-open races; only `close()` is ever reachable. */
  78. function abandonedHandleStub(): { handle: SessionHandle; close: ReturnType<typeof vi.fn> } {
  79. const close = vi.fn(async () => {})
  80. return { handle: { close } as unknown as SessionHandle, close }
  81. }
  82. function waitForIdle(ctx: Context, agent: Agent): Promise<void> {
  83. return new Promise((resolve) => {
  84. const dispose = ctx.on('agent/status', ({ agent: subject, status }) => {
  85. if (subject === agent && status === 'idle') { dispose(); resolve() }
  86. })
  87. })
  88. }
  89. /** Fail a lifecycle regression promptly instead of waiting for Vitest's suite timeout. */
  90. async function promptly<T>(job: Promise<T>): Promise<T> {
  91. const timeout = Promise.withResolvers<never>()
  92. const timer = setTimeout(() => { timeout.reject(new Error('lifecycle task did not settle promptly')) }, 1000)
  93. try {
  94. return await Promise.race([job, timeout.promise])
  95. } finally {
  96. clearTimeout(timer)
  97. }
  98. }
  99. /** Throw an arbitrary callback value to exercise the public unknown-error boundary. */
  100. function throwUnknown(value: unknown): never {
  101. throw value
  102. }
  103. describe('the session-persistence Agent Note: AgentLoop factory create/resume', () => {
  104. it('normalizes a non-Error resume publication failure for rollback and releases the write handle', async () => {
  105. const sessionId = SessionId('unknown-resume-failure-s')
  106. const root = await persistSession(sessionId)
  107. const ctx = await mountPersistentHarness(root, new MockAdapter([textResponse('next')]))
  108. const failure = { source: 'resume' }
  109. ctx.on('session/created', () => throwUnknown(failure))
  110. await expect(ctx.agents.resume({
  111. resumeSessionId: sessionId,
  112. })).rejects.toBe(failure)
  113. expect(ctx.agents.get(SessionId('unknown-resume-failure'))).toBeUndefined()
  114. expect(ctx.sessions.get(sessionId)).toBeUndefined()
  115. // Rollback closed the write handle: write ownership is claimable again.
  116. const reopened = await ctx.sessionPersistence.open(sessionId, 'write')
  117. await reopened.close()
  118. await ctx.fiber.dispose()
  119. })
  120. it('createAgent uses the caller-supplied sessionId (not ${id}-session)', async () => {
  121. const adapter = new MockAdapter([textResponse('hi')])
  122. const { ctx } = await persistentHarness(adapter)
  123. const { agent } = await ctx.agents.create({ sessionId: SessionId('custom-session'), meta: { cwd: '/w' } })
  124. expect(agent.session.id).toBe('custom-session')
  125. expect(agent.session.header.cwd).toBe('/w')
  126. await ctx.fiber.dispose()
  127. })
  128. it('createAgent rejects a duplicate identity without orphaning a session', async () => {
  129. const adapter = new MockAdapter([textResponse('hi')])
  130. const { ctx } = await persistentHarness(adapter)
  131. const sessionId = SessionId('sess-a')
  132. await ctx.agents.create({ sessionId })
  133. await expect(ctx.agents.create({ sessionId })).rejects.toThrow(/already exists/)
  134. expect(ctx.sessions.list()).toHaveLength(1)
  135. await ctx.fiber.dispose()
  136. })
  137. it('a created agent stores its seed and live turn durably through its handle', async () => {
  138. const seed: SessionEvent[] = [
  139. { type: 'turn/start', seq: SessionSeq(0), time: 1, data: { turn: 1 } },
  140. { type: 'turn/end', seq: SessionSeq(1), time: 2, data: { turn: 1, reason: { kind: 'completed' } } },
  141. ]
  142. const { ctx } = await persistentHarness(new MockAdapter([textResponse('stored')]))
  143. const sessionId = SessionId('durable-create')
  144. const handle = await ctx.agents.create({
  145. sessionId,
  146. seed,
  147. meta: { cwd: '/w' },
  148. agentOptions: { provider: 'mock', model: 'mock' },
  149. })
  150. handle.agent.followup(createUserMessage({ content: [{ type: 'text', text: 'q' }], source: { kind: 'user' } }))
  151. await waitForIdle(ctx, handle.agent)
  152. await handle.dispose()
  153. const stored = await readStoredEvents(ctx, sessionId)
  154. const seqs = stored.map(event => event.seq)
  155. expect(seqs).toEqual(seqs.map((_, index) => index))
  156. // The constructor seed (turn 1 + its end-seed marker) precedes the live turn 2.
  157. expect(stored.slice(0, 2).map(event => event.type)).toEqual(['turn/start', 'turn/end'])
  158. expect(stored[2]?.type).toBe('session/end-seed')
  159. const turnStarts = stored.filter(event => event.type === 'turn/start')
  160. expect(turnStarts.map(event => event.type === 'turn/start' && event.data.turn)).toEqual([1, 2])
  161. expect(stored.some(event => event.type === 'turn/end' && event.data.turn === 2)).toBe(true)
  162. expect(JSON.stringify(stored)).toContain('stored')
  163. await ctx.fiber.dispose()
  164. })
  165. it('a rejecting final writer close releases the registries, then rejects disposal', async () => {
  166. const { ctx } = await persistentHarness(new MockAdapter([textResponse('hi')]))
  167. const sessionId = SessionId('drain-close-fails')
  168. const handle = await ctx.agents.create({ sessionId })
  169. const persisted = await ctx.sessionPersistence.open(sessionId, 'read')
  170. await persisted.close()
  171. const stored = [...(ctx.sessionPersistence as unknown as {
  172. tracker: { openHandles: Set<SessionHandle> }
  173. }).tracker.openHandles].find(open => open.id === sessionId && open.access === 'write')
  174. if (stored === undefined) throw new Error('missing owned write handle')
  175. // The real close still runs (releasing write ownership); the injected
  176. // failure models a drain that reports a durability error at close.
  177. const realClose = stored.close.bind(stored)
  178. vi.spyOn(stored, 'close').mockImplementation(async () => {
  179. await realClose()
  180. throw new Error('close exploded')
  181. })
  182. await expect(handle.dispose()).rejects.toThrow('close exploded')
  183. // Teardown reached quiescence before the rejection: the agent and session
  184. // are unregistered, and write ownership is released — the never-flushed
  185. // session reports absence, not an ownership conflict.
  186. expect(ctx.agents.get(sessionId)).toBeUndefined()
  187. expect(ctx.sessions.get(sessionId)).toBeUndefined()
  188. await expect(ctx.sessionPersistence.open(sessionId, 'write')).rejects.toThrow('not found')
  189. await ctx.fiber.dispose()
  190. })
  191. it('combines a machine-teardown failure with a close failure into one rejection', async () => {
  192. const { ctx } = await persistentHarness(new MockAdapter([textResponse('hi')]))
  193. const sessionId = SessionId('drain-both-fail')
  194. const handle = await ctx.agents.create({ sessionId })
  195. const stored = [...(ctx.sessionPersistence as unknown as {
  196. tracker: { openHandles: Set<SessionHandle> }
  197. }).tracker.openHandles].find(open => open.id === sessionId && open.access === 'write')
  198. if (stored === undefined) throw new Error('missing owned write handle')
  199. const machine = handle.agent as Agent & { scope: { dispose: () => Promise<void> } }
  200. vi.spyOn(machine.scope, 'dispose').mockRejectedValue(new Error('scope exploded'))
  201. vi.spyOn(stored, 'close').mockRejectedValue(new Error('close exploded'))
  202. const failure = await handle.dispose().then(() => undefined, (error: unknown) => error)
  203. if (!(failure instanceof AggregateError)) throw new Error('expected an AggregateError rejection')
  204. expect(failure.message).toContain(`agent "${sessionId}" disposal failed`)
  205. expect(failure.errors.map(error => (error as Error).message)).toEqual(['scope exploded', 'close exploded'])
  206. expect(ctx.agents.get(sessionId)).toBeUndefined()
  207. await ctx.fiber.dispose()
  208. })
  209. it('agent dispose releases write ownership of its stored session', async () => {
  210. const { ctx } = await persistentHarness(new MockAdapter([textResponse('hi')]))
  211. const sessionId = SessionId('ownership-release')
  212. const handle = await ctx.agents.create({ sessionId })
  213. await expect(ctx.sessionPersistence.open(sessionId, 'write'))
  214. .rejects.toThrow(/already owned by an active write handle/)
  215. // Materialize the log so the session outlives its creator handle.
  216. handle.agent.followup(createUserMessage({ content: [{ type: 'text', text: 'q' }], source: { kind: 'user' } }))
  217. await waitForIdle(ctx, handle.agent)
  218. await handle.dispose()
  219. const reopened = await ctx.sessionPersistence.open(sessionId, 'write')
  220. expect(reopened.access).toBe('write')
  221. await reopened.close()
  222. await ctx.fiber.dispose()
  223. })
  224. it('a stored-session create failure rolls the fresh identity back', async () => {
  225. const { ctx } = await persistentHarness(new MockAdapter([]))
  226. const sessionId = SessionId('create-backend-fail')
  227. vi.spyOn(ctx.sessionPersistence, 'create').mockRejectedValueOnce(new Error('backend create failed'))
  228. await expect(ctx.agents.create({ sessionId })).rejects.toThrow('backend create failed')
  229. expect(ctx.agents.get(sessionId)).toBeUndefined()
  230. expect(ctx.sessions.get(sessionId)).toBeUndefined()
  231. // The identity was fully released: the same id creates cleanly afterwards.
  232. const retry = await ctx.agents.create({ sessionId })
  233. await retry.dispose()
  234. await ctx.fiber.dispose()
  235. })
  236. it('a seed append failure closes the fresh write handle and rethrows', async () => {
  237. const { ctx } = await persistentHarness(new MockAdapter([]))
  238. const sessionId = SessionId('seed-append-fail')
  239. const seed: SessionEvent[] = [
  240. { type: 'turn/start', seq: SessionSeq(0), time: 1, data: { turn: 1 } },
  241. { type: 'turn/end', seq: SessionSeq(1), time: 2, data: { turn: 1, reason: { kind: 'completed' } } },
  242. ]
  243. const originalCreate = ctx.sessionPersistence.create.bind(ctx.sessionPersistence)
  244. let closeSpy: MockInstance<() => Promise<void>> | undefined
  245. ctx.sessionPersistence.create = async (header, options) => {
  246. const handle = await originalCreate(header, options)
  247. vi.spyOn(handle, 'append').mockRejectedValue(new Error('seed append failed'))
  248. closeSpy = vi.spyOn(handle, 'close')
  249. return handle
  250. }
  251. await expect(ctx.agents.create({ sessionId, seed })).rejects.toThrow('seed append failed')
  252. expect(closeSpy).toHaveBeenCalled()
  253. expect(ctx.agents.get(sessionId)).toBeUndefined()
  254. expect(ctx.sessions.get(sessionId)).toBeUndefined()
  255. // The closed never-materialized creation left no stored session behind.
  256. await expect(ctx.sessionPersistence.open(sessionId, 'write'))
  257. .rejects.toThrow(/not found/)
  258. await ctx.fiber.dispose()
  259. })
  260. it('a setup failure before publication leaves no stored residue; the id creates again', async () => {
  261. const { ctx } = await persistentHarness(new MockAdapter([]))
  262. const sessionId = SessionId('setup-fail-no-residue')
  263. const seed: SessionEvent[] = [
  264. { type: 'turn/start', seq: SessionSeq(0), time: 1, data: { turn: 1 } },
  265. { type: 'turn/end', seq: SessionSeq(1), time: 2, data: { turn: 1, reason: { kind: 'completed' } } },
  266. ]
  267. await expect(ctx.agents.create({
  268. sessionId,
  269. seed,
  270. setup: () => { throw new Error('setup refused') },
  271. })).rejects.toThrow('setup refused')
  272. // The seed is stored only at the publication commit point, so the failed
  273. // attempt materialized nothing and released the identity completely.
  274. await expect(ctx.sessionPersistence.stat(sessionId)).resolves.toBeUndefined()
  275. const retry = await ctx.agents.create({ sessionId, seed })
  276. await expect(ctx.sessionPersistence.stat(sessionId)).resolves.toBeDefined()
  277. await retry.dispose()
  278. await ctx.fiber.dispose()
  279. })
  280. it('rollback swallows a rejecting handle close after a prepare failure (create and createAgent)', async () => {
  281. const { ctx } = await persistentHarness(new MockAdapter([]))
  282. const originalCreate = ctx.sessionPersistence.create.bind(ctx.sessionPersistence)
  283. const closeSpies: Array<{ mockRestore: () => void }> = []
  284. ctx.sessionPersistence.create = async (header, options) => {
  285. const handle = await originalCreate(header, options)
  286. closeSpies.push(vi.spyOn(handle, 'close').mockRejectedValue(new Error('close failed')))
  287. return handle
  288. }
  289. // Config create path: prepare's option validation throws after the handle exists.
  290. await expect(ctx.agentLoop.create(SessionId('close-reject-config'), { maxTokens: -1 }))
  291. .rejects.toThrow('agent maxTokens must be a positive safe integer')
  292. expect(ctx.agents.get(SessionId('close-reject-config'))).toBeUndefined()
  293. // Owned createAgent path: the same validation failure after the handle exists.
  294. await expect(ctx.agents.create({
  295. sessionId: SessionId('close-reject-owned'),
  296. agentOptions: { maxTokens: -1 },
  297. })).rejects.toThrow('agent maxTokens must be a positive safe integer')
  298. expect(ctx.agents.get(SessionId('close-reject-owned'))).toBeUndefined()
  299. for (const spy of closeSpies.splice(0)) spy.mockRestore()
  300. await ctx.fiber.dispose()
  301. })
  302. it('a reentrant abort during preparation swallows a rejecting handle close', async () => {
  303. const { ctx } = await persistentHarness(new MockAdapter([]))
  304. const sessionId = SessionId('prepare-abort-close-reject')
  305. const originalCreate = ctx.sessionPersistence.create.bind(ctx.sessionPersistence)
  306. const spies: Array<{ mockRestore: () => void }> = []
  307. let closed: Promise<void> | undefined
  308. ctx.sessionPersistence.create = async (header, options) => {
  309. const handle = await originalCreate(header, options)
  310. const realClose = handle.close.bind(handle)
  311. spies.push(vi.spyOn(handle, 'close').mockImplementation(async () => {
  312. closed = realClose()
  313. await closed
  314. throw new Error('close failed')
  315. }))
  316. return handle
  317. }
  318. const reason = new Error('cancelled while preparing')
  319. const controller = new AbortController()
  320. let aborted = false
  321. ctx.on('internal/plugin', (fiber) => {
  322. if (aborted || fiber.name !== 'scope') return
  323. aborted = true
  324. controller.abort(reason)
  325. })
  326. await expect(ctx.agents.create({ sessionId, signal: controller.signal })).rejects.toBe(reason)
  327. // The rejecting close stays swallowed by the rollback; wait for the
  328. // rollback's real close so factory teardown finds a quiescent lifecycle.
  329. await vi.waitFor(() => { if (closed === undefined) throw new Error('close not reached') })
  330. await closed
  331. expect(ctx.agents.get(sessionId)).toBeUndefined()
  332. for (const spy of spies.splice(0)) spy.mockRestore()
  333. await ctx.fiber.dispose()
  334. })
  335. it('config create rollback swallows a rejecting close after a publish failure', async () => {
  336. const { ctx } = await persistentHarness(new MockAdapter([]))
  337. const sessionId = SessionId('config-publish-close-reject')
  338. const originalCreate = ctx.sessionPersistence.create.bind(ctx.sessionPersistence)
  339. const spies: Array<{ mockRestore: () => void }> = []
  340. let closed: Promise<void> | undefined
  341. ctx.sessionPersistence.create = async (header, options) => {
  342. const handle = await originalCreate(header, options)
  343. const realClose = handle.close.bind(handle)
  344. spies.push(vi.spyOn(handle, 'close').mockImplementation(async () => {
  345. closed = realClose()
  346. await closed
  347. throw new Error('close failed')
  348. }))
  349. return handle
  350. }
  351. const announce = vi.spyOn(ctx.agents, 'announce').mockImplementation(() => {
  352. throw new Error('announce failed')
  353. })
  354. await expect(ctx.agentLoop.create(sessionId)).rejects.toThrow('announce failed')
  355. announce.mockRestore()
  356. await vi.waitFor(() => { if (closed === undefined) throw new Error('close not reached') })
  357. await closed
  358. expect(ctx.agents.get(sessionId)).toBeUndefined()
  359. for (const spy of spies.splice(0)) spy.mockRestore()
  360. await ctx.fiber.dispose()
  361. })
  362. it('a seed append failure swallows a rejecting handle close', async () => {
  363. const { ctx } = await persistentHarness(new MockAdapter([]))
  364. const sessionId = SessionId('seed-append-close-reject')
  365. const seed: SessionEvent[] = [
  366. { type: 'turn/start', seq: SessionSeq(0), time: 1, data: { turn: 1 } },
  367. { type: 'turn/end', seq: SessionSeq(1), time: 2, data: { turn: 1, reason: { kind: 'completed' } } },
  368. ]
  369. const originalCreate = ctx.sessionPersistence.create.bind(ctx.sessionPersistence)
  370. const spies: Array<{ mockRestore: () => void }> = []
  371. ctx.sessionPersistence.create = async (header, options) => {
  372. const handle = await originalCreate(header, options)
  373. spies.push(vi.spyOn(handle, 'append').mockRejectedValue(new Error('seed append failed')))
  374. spies.push(vi.spyOn(handle, 'close').mockRejectedValue(new Error('close failed')))
  375. return handle
  376. }
  377. await expect(ctx.agents.create({ sessionId, seed })).rejects.toThrow('seed append failed')
  378. expect(ctx.agents.get(sessionId)).toBeUndefined()
  379. expect(ctx.sessions.get(sessionId)).toBeUndefined()
  380. for (const spy of spies.splice(0)) spy.mockRestore()
  381. await ctx.fiber.dispose()
  382. })
  383. it('a resume read failure swallows a rejecting handle close during rollback', async () => {
  384. const sessionId = SessionId('resume-read-close-reject')
  385. const root = await persistSession(sessionId)
  386. const ctx = await mountPersistentHarness(root, new MockAdapter([]))
  387. const originalOpen = ctx.sessionPersistence.open.bind(ctx.sessionPersistence)
  388. const spies: Array<{ mockRestore: () => void }> = []
  389. ctx.sessionPersistence.open = async (id, access, options) => {
  390. const handle = await originalOpen(id, access, options)
  391. spies.push(vi.spyOn(handle, 'read').mockRejectedValue(new Error('stored read failed')))
  392. spies.push(vi.spyOn(handle, 'close').mockRejectedValue(new Error('close failed')))
  393. return handle
  394. }
  395. await expect(ctx.agents.resume({ resumeSessionId: sessionId }))
  396. .rejects.toThrow('stored read failed')
  397. expect(ctx.agents.get(sessionId)).toBeUndefined()
  398. expect(ctx.sessions.get(sessionId)).toBeUndefined()
  399. for (const spy of spies.splice(0)) spy.mockRestore()
  400. await ctx.fiber.dispose()
  401. })
  402. it('resume cannot crash-repair a turn owned by a live agent', async () => {
  403. const { ctx } = await persistentHarness(new MockAdapter([textResponse('unused')]))
  404. const sessionId = SessionId('live-resume-race')
  405. const first = (await ctx.agents.create({ sessionId })).agent
  406. first.session.append('turn/start', { turn: 1 })
  407. await ctx.sessions.flush(first.session)
  408. await expect(ctx.agents.resume({ resumeSessionId: sessionId }))
  409. .rejects.toThrow(/already owned by an active write handle/)
  410. first.session.append('turn/end', { turn: 1, reason: { kind: 'completed' } })
  411. await ctx.sessions.flush(first.session)
  412. const stored = await readStoredEvents(ctx, sessionId)
  413. expect(stored.map(event => event.type)).toEqual(['turn/start', 'turn/end'])
  414. expect(stored.at(-1)).toMatchObject({
  415. type: 'turn/end',
  416. data: { reason: { kind: 'completed' } },
  417. })
  418. await ctx.fiber.dispose()
  419. })
  420. it('resume appends interrupted-turn closers durably through the handle', async () => {
  421. // Lifecycle 1: store an interrupted log — an open turn with no turn/end.
  422. const sessionId = SessionId('interrupted-resume')
  423. const { ctx: ctx1, root } = await persistentHarness(new MockAdapter([]))
  424. await seedStoredSession(ctx1, sessionId, [
  425. { type: 'turn/start', seq: SessionSeq(0), time: 1, data: { turn: 1 } },
  426. ])
  427. // A read returns the PHYSICAL validated log: no synthetic closers.
  428. const raw = await readStoredEvents(ctx1, sessionId)
  429. expect(raw.map(event => event.type)).toEqual(['turn/start'])
  430. await ctx1.fiber.dispose()
  431. // Lifecycle 2: resume repairs the tail and stores the repair durably.
  432. const ctx2 = await mountPersistentHarness(root, new MockAdapter([]))
  433. const handle = await ctx2.agents.resume({ resumeSessionId: sessionId })
  434. expect(handle.agent.session.snapshotEvents().map(event => event.type))
  435. .toEqual(['turn/start', 'turn/end', 'session/end-seed'])
  436. await handle.dispose()
  437. const stored = await readStoredEvents(ctx2, sessionId)
  438. expect(stored.map(event => event.type)).toEqual(['turn/start', 'turn/end', 'session/end-seed'])
  439. expect(stored[1]).toMatchObject({ data: { reason: { kind: 'interrupted' } } })
  440. await ctx2.fiber.dispose()
  441. })
  442. it('resume closes an interrupted tool call durably: tool/result, step/end, turn/end', async () => {
  443. const sessionId = SessionId('interrupted-tool-resume')
  444. const { ctx: ctx1, root } = await persistentHarness(new MockAdapter([]))
  445. await seedStoredSession(ctx1, sessionId, [
  446. { type: 'turn/start', seq: SessionSeq(0), time: 1, data: { turn: 1 } },
  447. { type: 'step/start', seq: SessionSeq(1), time: 1, data: { turn: 1, step: 1 } },
  448. { type: 'assistant/message', seq: SessionSeq(2), time: 2, surfaceOp: 'append', data: {
  449. turn: 1, step: 1,
  450. stream: [],
  451. message: createMessage({
  452. role: 'assistant',
  453. content: [{ type: 'tool-call', id: ToolCallId('call-1'), name: 'bash', arguments: '{}' }],
  454. source: { kind: 'model', provider: 'mock', model: 'mock' },
  455. }),
  456. } },
  457. { type: 'tool/call', seq: SessionSeq(3), time: 2, data: { turn: 1, step: 1, callId: ToolCallId('call-1'), name: 'bash', arguments: '{}' } },
  458. ] as SessionEvent[])
  459. await ctx1.fiber.dispose()
  460. const ctx2 = await mountPersistentHarness(root, new MockAdapter([]))
  461. const handle = await ctx2.agents.resume({ resumeSessionId: sessionId })
  462. await handle.dispose()
  463. // The multi-closer set lands durably in one contiguous batch, and the
  464. // synthetic tool/result cites the recorded tool/call seq.
  465. const stored = await readStoredEvents(ctx2, sessionId)
  466. expect(stored.map(event => event.type)).toEqual([
  467. 'turn/start', 'step/start', 'assistant/message', 'tool/call',
  468. 'tool/result', 'step/end', 'turn/end', 'session/end-seed',
  469. ])
  470. expect(stored.map(event => event.seq)).toEqual([0, 1, 2, 3, 4, 5, 6, 7])
  471. expect(stored[4]).toMatchObject({
  472. sourceEventSeqs: [3],
  473. data: { error: { code: TOOL_OUTCOME_UNKNOWN } },
  474. })
  475. expect(stored[6]).toMatchObject({ data: { reason: { kind: 'interrupted' } } })
  476. await ctx2.fiber.dispose()
  477. })
  478. it('resume over a torn physical tail continues from the committed prefix', async () => {
  479. const sessionId = SessionId('torn-tail-resume')
  480. const root = await mkdtemp(join(tmpdir(), 'dsh-resume-torn-'))
  481. dirs.push(root)
  482. const ctx1 = await mountPersistentHarness(root, new MockAdapter([]), 'none')
  483. await seedStoredSession(ctx1, sessionId, [
  484. { type: 'turn/start', seq: SessionSeq(0), time: 1, data: { turn: 1 } },
  485. ])
  486. await ctx1.fiber.dispose()
  487. // Crash artifact: a half-written record with no trailing newline.
  488. const logs = (await readdir(root, { recursive: true })).filter(name => name.endsWith('.jsonl'))
  489. expect(logs).toHaveLength(1)
  490. await appendFile(join(root, logs[0] as string), '{"type":"assistant/chunk","seq":1,"ti')
  491. // Resume truncates the torn tail under its write open, then appends the
  492. // closers immediately after the committed prefix — no gap, no fragment.
  493. const ctx2 = await mountPersistentHarness(root, new MockAdapter([]), 'none')
  494. const handle = await ctx2.agents.resume({ resumeSessionId: sessionId })
  495. expect(handle.agent.session.snapshotEvents().map(event => event.type))
  496. .toEqual(['turn/start', 'turn/end', 'session/end-seed'])
  497. await handle.dispose()
  498. const stored = await readStoredEvents(ctx2, sessionId)
  499. expect(stored.map(event => `${event.type}@${event.seq}`))
  500. .toEqual(['turn/start@0', 'turn/end@1', 'session/end-seed@2'])
  501. await ctx2.fiber.dispose()
  502. })
  503. it('createAgent works without meta (no cwd)', async () => {
  504. const adapter = new MockAdapter([textResponse('hi')])
  505. const { ctx } = await persistentHarness(adapter)
  506. const { agent } = await ctx.agents.create({ sessionId: SessionId('nometa-session') })
  507. expect(agent.session.id).toBe('nometa-session')
  508. expect(agent.session.header.cwd).toBeUndefined()
  509. await ctx.fiber.dispose()
  510. })
  511. it('resume of a session with no cwd carries an undefined cwd header', async () => {
  512. // Lifecycle 1: create a no-cwd session and run a turn.
  513. const adapter1 = new MockAdapter([textResponse('a')])
  514. const { ctx: ctx1, root } = await persistentHarness(adapter1)
  515. const h1 = await ctx1.agents.create({ sessionId: SessionId('nocwd-sess') })
  516. h1.agent.followup(createUserMessage({ content: [{ type: 'text', text: 'q' }], source: { kind: 'user' } }))
  517. await waitForIdle(ctx1, h1.agent)
  518. // Agent disposal drains the writer through the still-open handle.
  519. await h1.dispose()
  520. await ctx1.fiber.dispose()
  521. // Lifecycle 2: resume it; the header cwd stays undefined (no-cwd branch).
  522. const ctx2 = await mountPersistentHarness(root, new MockAdapter([textResponse('b')]))
  523. const a2 = (await ctx2.agents.resume({ resumeSessionId: SessionId('nocwd-sess') })).agent
  524. expect(a2.session.header.cwd).toBeUndefined()
  525. await ctx2.fiber.dispose()
  526. })
  527. it('agent/created fires "startup" for createAgent and "resume" for resume()', async () => {
  528. // Lifecycle 1: a fresh createAgent announces creation with source 'startup'.
  529. const adapter1 = new MockAdapter([textResponse('a')])
  530. const { ctx: ctx1, root } = await persistentHarness(adapter1)
  531. const sources1: string[] = []
  532. ctx1.on('agent/created', ({ source }) => void sources1.push(source))
  533. const h1 = await ctx1.agents.create({ sessionId: SessionId('start-sess') })
  534. expect(sources1).toEqual(['startup'])
  535. h1.agent.followup(createUserMessage({ content: [{ type: 'text', text: 'q' }], source: { kind: 'user' } }))
  536. await waitForIdle(ctx1, h1.agent)
  537. await h1.dispose()
  538. await ctx1.fiber.dispose()
  539. // Lifecycle 2: resuming the persisted session announces creation 'resume'.
  540. const ctx2 = await mountPersistentHarness(root, new MockAdapter([textResponse('b')]))
  541. const sources2: string[] = []
  542. ctx2.on('agent/created', ({ source }) => void sources2.push(source))
  543. await ctx2.agents.resume({ resumeSessionId: SessionId('start-sess') })
  544. expect(sources2).toEqual(['resume'])
  545. await ctx2.fiber.dispose()
  546. })
  547. it('resume awaits setup while unpublished, then publishes a fully composed world in order', async () => {
  548. const sessionId = SessionId('resume-setup-success')
  549. const root = await persistSession(sessionId)
  550. const ctx = await mountPersistentHarness(root, new MockAdapter([textResponse('next')]))
  551. const gate = Promise.withResolvers<undefined>()
  552. const setupStarted = Promise.withResolvers<undefined>()
  553. const order: string[] = []
  554. ctx.on('session/created', (session) => {
  555. expect(ctx.sessions.get(session.id)).toBe(session)
  556. expect(ctx.agents.get(sessionId)?.session).toBe(session)
  557. order.push('session/created')
  558. })
  559. ctx.on('agent/created', ({ agent }) => {
  560. expect(agent.status).toBe('idle')
  561. expect(() => { agent.cancel({ kind: 'user' }) }).not.toThrow()
  562. order.push('agent/created')
  563. })
  564. const resuming = ctx.agents.resume({
  565. resumeSessionId: sessionId,
  566. agentOptions: { provider: 'mock', model: 'mock' },
  567. setup: async (agentCtx, agent) => {
  568. expect(agent.id).toBe(sessionId)
  569. // The two persisted events plus the end-seed marker.
  570. expect(agent.session.snapshotEvents()).toHaveLength(3)
  571. agentCtx.on('session/created', () => void order.push('setup-listener:session/created'))
  572. agentCtx.on('agent/created', () => void order.push('setup-listener:agent/created'))
  573. order.push('setup:start')
  574. setupStarted.resolve(undefined)
  575. await gate.promise
  576. order.push('setup:end')
  577. return {
  578. commit: () => {
  579. expect(ctx.agents.get(sessionId)).toBeUndefined()
  580. expect(ctx.sessions.get(sessionId)).toBeUndefined()
  581. order.push('setup:commit')
  582. },
  583. }
  584. },
  585. })
  586. await setupStarted.promise
  587. expect(ctx.agents.get(sessionId)).toBeUndefined()
  588. expect(ctx.sessions.get(sessionId)).toBeUndefined()
  589. expect(order).toEqual(['setup:start'])
  590. gate.resolve(undefined)
  591. const handle = await resuming
  592. expect(order).toEqual([
  593. 'setup:start',
  594. 'setup:end',
  595. 'setup:commit',
  596. 'session/created',
  597. 'setup-listener:session/created',
  598. 'agent/created',
  599. 'setup-listener:agent/created',
  600. ])
  601. await handle.dispose()
  602. await ctx.fiber.dispose()
  603. })
  604. it('a resumed session stores its repair suffix durably before publication', async () => {
  605. // The stored fixture (two events, no end-seed) gains the end-seed marker
  606. // through the resume handle: after one resume lifecycle the STORED log
  607. // carries it, so the next resume reads it back without re-marking.
  608. const sessionId = SessionId('resume-suffix-durable')
  609. const root = await persistSession(sessionId)
  610. const ctx = await mountPersistentHarness(root, new MockAdapter([]))
  611. const first = await ctx.agents.resume({ resumeSessionId: sessionId })
  612. await first.dispose()
  613. const stored = await readStoredEvents(ctx, sessionId)
  614. expect(stored.map(event => event.type)).toEqual(['turn/start', 'turn/end', 'session/end-seed'])
  615. const second = await ctx.agents.resume({ resumeSessionId: sessionId })
  616. expect(second.agent.session.snapshotEvents().map(event => event.type))
  617. .toEqual(['turn/start', 'turn/end', 'session/end-seed'])
  618. await second.dispose()
  619. const restored = await readStoredEvents(ctx, sessionId)
  620. expect(restored.map(event => event.type)).toEqual(['turn/start', 'turn/end', 'session/end-seed'])
  621. await ctx.fiber.dispose()
  622. })
  623. it('successful resume disposal retires its caller-owned transaction effects', async () => {
  624. const sessionId = SessionId('resume-retired-effects-s')
  625. const root = await persistSession(sessionId)
  626. const ctx = await mountPersistentHarness(root, new MockAdapter([textResponse('next')]))
  627. const handle = await ctx.agents.resume({
  628. resumeSessionId: sessionId,
  629. agentOptions: { provider: 'mock', model: 'mock' },
  630. })
  631. const transactionLabels = [`agentLoop.lifecycle(${sessionId})`]
  632. expect(ctx.fiber.getEffects().map(effect => effect.label)).toEqual(expect.arrayContaining(transactionLabels))
  633. await handle.dispose()
  634. expect(ctx.fiber.getEffects().filter(effect => transactionLabels.includes(effect.label))).toEqual([])
  635. await ctx.fiber.dispose()
  636. })
  637. it('resume setup rejection publishes nothing, unwinds, and releases the identity', async () => {
  638. const sessionId = SessionId('resume-setup-reject')
  639. const root = await persistSession(sessionId)
  640. const ctx = await mountPersistentHarness(root, new MockAdapter([textResponse('next')]))
  641. const published: string[] = []
  642. ctx.on('session/created', () => void published.push('session/created'))
  643. ctx.on('agent/created', () => void published.push('agent/created'))
  644. await expect(ctx.agents.resume({
  645. resumeSessionId: sessionId,
  646. agentOptions: { provider: 'mock', model: 'mock' },
  647. setup: async () => {
  648. await Promise.resolve()
  649. throw new Error('resume setup failed')
  650. },
  651. })).rejects.toThrow('resume setup failed')
  652. expect(published).toEqual([])
  653. expect(ctx.agents.get(sessionId)).toBeUndefined()
  654. expect(ctx.sessions.get(sessionId)).toBeUndefined()
  655. const retry = await ctx.agents.resume({
  656. resumeSessionId: sessionId,
  657. agentOptions: { provider: 'mock', model: 'mock' },
  658. })
  659. await retry.dispose()
  660. await ctx.fiber.dispose()
  661. })
  662. it('resume setup commit rejection publishes nothing and releases the identity', async () => {
  663. const sessionId = SessionId('resume-setup-commit-reject')
  664. const root = await persistSession(sessionId)
  665. const ctx = await mountPersistentHarness(root, new MockAdapter([textResponse('next')]))
  666. const published: string[] = []
  667. ctx.on('session/created', () => void published.push('session/created'))
  668. ctx.on('agent/created', () => void published.push('agent/created'))
  669. await expect(ctx.agents.resume({
  670. resumeSessionId: sessionId,
  671. agentOptions: { provider: 'mock', model: 'mock' },
  672. setup: () => ({
  673. commit: () => { throw new Error('resume setup commit failed') },
  674. }),
  675. })).rejects.toThrow('resume setup commit failed')
  676. expect(published).toEqual([])
  677. expect(ctx.agents.get(sessionId)).toBeUndefined()
  678. expect(ctx.sessions.get(sessionId)).toBeUndefined()
  679. const retry = await ctx.agents.resume({
  680. resumeSessionId: sessionId,
  681. agentOptions: { provider: 'mock', model: 'mock' },
  682. })
  683. await retry.dispose()
  684. await ctx.fiber.dispose()
  685. })
  686. it('owner unload aborts resume setup and cannot publish after the callback settles', async () => {
  687. const sessionId = SessionId('resume-setup-owner-unload')
  688. const root = await persistSession(sessionId)
  689. const ctx = await mountPersistentHarness(root, new MockAdapter([textResponse('next')]))
  690. const gate = Promise.withResolvers<undefined>()
  691. const setupStarted = Promise.withResolvers<undefined>()
  692. const published: string[] = []
  693. ctx.on('session/created', () => void published.push('session/created'))
  694. ctx.on('agent/created', () => void published.push('agent/created'))
  695. let resuming!: ReturnType<typeof ctx.agents.resume>
  696. const owner = await ctx.plugin(Object.assign((inner: Context) => {
  697. resuming = inner.agents.resume({
  698. resumeSessionId: sessionId,
  699. agentOptions: { provider: 'mock', model: 'mock' },
  700. setup: async () => {
  701. setupStarted.resolve(undefined)
  702. await gate.promise
  703. },
  704. })
  705. }, { inject: ['agents'] }))
  706. await setupStarted.promise
  707. await owner.dispose()
  708. await expect(resuming).rejects.toThrow(/owner disposed during setup/)
  709. expect(published).toEqual([])
  710. expect(ctx.agents.get(sessionId)).toBeUndefined()
  711. expect(ctx.sessions.get(sessionId)).toBeUndefined()
  712. gate.resolve(undefined)
  713. await Promise.resolve()
  714. expect(published).toEqual([])
  715. await ctx.fiber.dispose()
  716. })
  717. it('owner unload aborts a never-settling persistence open, releases the identity, and blocks late publication', async () => {
  718. const sessionId = SessionId('resume-load-owner-unload')
  719. const root = await persistSession(sessionId)
  720. const ctx = await mountPersistentHarness(root, new MockAdapter([textResponse('next')]))
  721. const lateOpen = Promise.withResolvers<SessionHandle>()
  722. const openStarted = Promise.withResolvers<undefined>()
  723. const originalOpen = ctx.sessionPersistence.open.bind(ctx.sessionPersistence)
  724. let opens = 0
  725. ctx.sessionPersistence.open = (id, access, options) => {
  726. expect(id).toBe(sessionId)
  727. opens += 1
  728. if (opens === 1) {
  729. openStarted.resolve(undefined)
  730. return lateOpen.promise
  731. }
  732. return originalOpen(id, access, options)
  733. }
  734. const published: string[] = []
  735. ctx.on('session/created', () => void published.push('session/created'))
  736. ctx.on('agent/created', () => void published.push('agent/created'))
  737. let resuming!: ReturnType<typeof ctx.agents.resume>
  738. const owner = await ctx.plugin(Object.assign((inner: Context) => {
  739. resuming = inner.agents.resume({ resumeSessionId: sessionId, agentOptions: { provider: 'mock', model: 'mock' } })
  740. }, { inject: ['agents'] }))
  741. await openStarted.promise
  742. const rejection = expect(promptly(resuming)).rejects.toThrow(/owner disposed during setup/)
  743. await promptly(owner.dispose())
  744. expect(published).toEqual([])
  745. expect(ctx.agents.get(sessionId)).toBeUndefined()
  746. expect(ctx.sessions.get(sessionId)).toBeUndefined()
  747. // owner.dispose() awaited transaction settlement, so the same identities
  748. // can be reused before awaiting the public rejection.
  749. const retry = await promptly(ctx.agents.resume({ resumeSessionId: sessionId, agentOptions: { provider: 'mock', model: 'mock' } }))
  750. await rejection
  751. expect(opens).toBe(2)
  752. expect(published).toEqual(['session/created', 'agent/created'])
  753. // Settlement of the abandoned backend open cannot resume the old
  754. // transaction: the late handle is closed, and no second publication lands
  755. // after the retry owns the ids.
  756. const abandoned = abandonedHandleStub()
  757. lateOpen.resolve(abandoned.handle)
  758. await expect.poll(() => abandoned.close.mock.calls.length).toBe(1)
  759. expect(ctx.agents.get(sessionId)).toBe(retry.agent)
  760. expect(ctx.sessions.get(sessionId)).toBe(retry.agent.session)
  761. expect(published).toEqual(['session/created', 'agent/created'])
  762. await retry.dispose()
  763. await ctx.fiber.dispose()
  764. })
  765. it('AgentLoop unload aborts a never-settling persistence open and awaits wrapper settlement', async () => {
  766. const sessionId = SessionId('resume-load-factory-unload')
  767. const root = await persistSession(sessionId)
  768. const ctx = new Context()
  769. await ctx.plugin(LlmRuntime)
  770. await ctx.plugin(SessionStore)
  771. await ctx.plugin(SessionProjectionRegistry)
  772. await ctx.plugin(SystemPrompt)
  773. await ctx.plugin(ToolRuntime)
  774. await ctx.plugin(AgentRegistry)
  775. const loopFiber = await ctx.plugin(AgentLoop, { agents: [] })
  776. await ctx.plugin(JsonlSessionPersistence, { root })
  777. ctx.llm.registerAdapter(['mock'], new MockAdapter([textResponse('next')]))
  778. const lateOpen = Promise.withResolvers<SessionHandle>()
  779. const openStarted = Promise.withResolvers<undefined>()
  780. ctx.sessionPersistence.open = (id) => {
  781. expect(id).toBe(sessionId)
  782. openStarted.resolve(undefined)
  783. return lateOpen.promise
  784. }
  785. const published: string[] = []
  786. ctx.on('session/created', () => void published.push('session/created'))
  787. ctx.on('agent/created', () => void published.push('agent/created'))
  788. const resuming = ctx.agents.resume({ resumeSessionId: sessionId, agentOptions: { provider: 'mock', model: 'mock' } })
  789. await openStarted.promise
  790. const rejection = expect(promptly(resuming)).rejects.toThrow(/agent loop is not active/)
  791. await promptly(loopFiber.dispose())
  792. await rejection
  793. expect(published).toEqual([])
  794. expect(ctx.agents.get(sessionId)).toBeUndefined()
  795. expect(ctx.sessions.get(sessionId)).toBeUndefined()
  796. // The abandoned handle is closed once the hung open finally settles.
  797. const abandoned = abandonedHandleStub()
  798. lateOpen.resolve(abandoned.handle)
  799. await expect.poll(() => abandoned.close.mock.calls.length).toBe(1)
  800. expect(published).toEqual([])
  801. await ctx.fiber.dispose()
  802. })
  803. it('resume of a forked session preserves the lineage, seed boundary, and delegation depth in the header', async () => {
  804. // Lifecycle 1: persist a FORKED session (carries parentSession + seedLength
  805. // in its header) through createAgent with a complete-turn seed — the
  806. // factory stores the header and seed through its write handle.
  807. const seed: SessionEvent[] = [
  808. { type: 'turn/start', seq: SessionSeq(0), time: 1, data: { turn: 1 } },
  809. { type: 'turn/end', seq: SessionSeq(1), time: 2, data: { turn: 1, reason: { kind: 'completed' } } },
  810. ]
  811. const { ctx: ctx1, root } = await persistentHarness(new MockAdapter([]))
  812. const forked = await ctx1.agents.create({
  813. sessionId: SessionId('forked-sess'),
  814. seed,
  815. meta: { cwd: '/w', parentSession: SessionId('parent-sess'), isSeeded: true, delegationDepth: 1 },
  816. inheritedEventCount: SessionLogOffset(seed.length),
  817. })
  818. await forked.dispose()
  819. await ctx1.fiber.dispose()
  820. // Lifecycle 2: resume it; the parentSession + seedLength header survives the
  821. // round-trip (exercises resume's parentSession- and seedLength-present
  822. // branches). seedLength must come from the PERSISTED header, not from the
  823. // resume seed length (which is the whole stored log, not the original
  824. // boundary).
  825. const ctx2 = await mountPersistentHarness(root, new MockAdapter([textResponse('b')]))
  826. const a2 = (await ctx2.agents.resume({ resumeSessionId: SessionId('forked-sess') })).agent
  827. expect(a2.session.header.parentSession).toBe('parent-sess')
  828. expect(a2.session.header.cwd).toBe('/w')
  829. expect(a2.session.header.isSeeded).toBe(true)
  830. expect(a2.session.inheritedEventCount).toBe(seed.length)
  831. // The recursion budget survives resume — a dropped depth would let a
  832. // resumed child delegate as if it were top-level.
  833. expect(a2.session.header.delegationDepth).toBe(1)
  834. await ctx2.fiber.dispose()
  835. })
  836. // The crash simulation removes the wedged lifecycle's lock file, which only
  837. // POSIX's orphan-inode forfeit honors; Windows pins the name until the
  838. // process exits, and cross-process crash release is pinned by the jsonl
  839. // two-process e2e.
  840. it.skipIf(process.platform === 'win32')('a pending idle inject() survives persist + resume without a synthetic turn', async () => {
  841. const adapter1 = new MockAdapter([textResponse('answer')])
  842. const { ctx: ctx1, root } = await persistentHarness(adapter1)
  843. const a1 = (await ctx1.agents.create({ sessionId: SessionId('inject-sess'), meta: { cwd: '/w' }, agentOptions: { provider: 'mock', model: 'mock' } })).agent
  844. a1.followup(createUserMessage({ content: [{ type: 'text', text: 'q' }], source: { kind: 'user' } }))
  845. await waitForIdle(ctx1, a1)
  846. a1.inject(createUserMessage({ content: [{ type: 'text', text: 'background job 42 finished' }], source: { kind: 'plugin', plugin: 'tool-bash' } }))
  847. await a1.whenIdle()
  848. await ctx1.sessions.flush(a1.session)
  849. // Simulate a wedged first lifecycle: a graceful dispose would durably
  850. // discard the pending inject, and the still-open kernel write lock would
  851. // otherwise exclude the second lifecycle. Removing the lock file orphans
  852. // the held inode so the resumer locks a fresh one (the documented
  853. // forfeit-by-unlink escape hatch).
  854. await removeSessionLocks(root)
  855. // Lifecycle 2: resume; the injected context is still pending and becomes
  856. // model-visible when the next turn admits it.
  857. const ctx2 = await mountPersistentHarness(root, new MockAdapter([textResponse('next')]))
  858. const stored = await readStoredEvents(ctx2, SessionId('inject-sess'))
  859. expect(stored.some(event => event.type === 'agent/inbox/spliced')).toBe(true)
  860. expect(JSON.stringify(stored)).toContain('background job 42 finished')
  861. const a2 = (await ctx2.agents.resume({ resumeSessionId: SessionId('inject-sess'), agentOptions: { provider: 'mock', model: 'mock' } })).agent
  862. expect(JSON.stringify(a2.inbox.nextStep)).toContain('background job 42 finished')
  863. a2.followup(createUserMessage({ content: [{ type: 'text', text: 'continue' }], source: { kind: 'user' } }))
  864. await waitForIdle(ctx2, a2)
  865. const flat = JSON.stringify(a2.session.deriveMessages())
  866. expect(flat).toContain('background job 42 finished')
  867. await ctx2.fiber.dispose()
  868. await ctx1.fiber.dispose()
  869. })
  870. it('resume reloads a persisted session: history + turn numbering continue, no duplicate seqs', async () => {
  871. // Lifecycle 1: run one full turn, persisting it.
  872. const adapter1 = new MockAdapter([textResponse('first answer')])
  873. const { ctx: ctx1, root } = await persistentHarness(adapter1)
  874. const h1 = await ctx1.agents.create({ sessionId: SessionId('sess-resume'), meta: { cwd: '/w' } })
  875. const a1 = h1.agent
  876. a1.followup(createUserMessage({ content: [{ type: 'text', text: 'first question' }], source: { kind: 'user' } }))
  877. await waitForIdle(ctx1, a1)
  878. const events1 = a1.session.snapshotEvents()
  879. const seqs1 = events1.map(e => e.seq)
  880. expect(seqs1).toEqual([...seqs1].sort((x, y) => x - y)) // contiguous
  881. await h1.dispose()
  882. await ctx1.fiber.dispose()
  883. // Lifecycle 2: a brand-new context over the SAME root; resume the session.
  884. const ctx2 = await mountPersistentHarness(root, new MockAdapter([textResponse('second answer')]))
  885. const a2 = (await ctx2.agents.resume({ resumeSessionId: SessionId('sess-resume') })).agent
  886. // The resumed session carries the prior history…
  887. expect(a2.session.id).toBe('sess-resume')
  888. // …followed by one end-seed event marking the constructor seed.
  889. expect(a2.session.snapshotEvents().length).toBe(events1.length + 1)
  890. expect(a2.session.firstLiveSeq).toBe(events1.length)
  891. expect(a2.session.snapshotEvents().at(-1)?.type).toBe('session/end-seed')
  892. const replay = Session.create(SessionId('replay'), events1)
  893. expect(a2.session.deriveMessages()).toEqual(replay.deriveMessages())
  894. // …and a new turn continues numbering (turn 2) with contiguous seqs.
  895. a2.followup(createUserMessage({ content: [{ type: 'text', text: 'second question' }], source: { kind: 'user' } }))
  896. await waitForIdle(ctx2, a2)
  897. const allSeqs = a2.session.snapshotEvents().map(e => e.seq)
  898. expect(allSeqs).toEqual(allSeqs.map((_, i) => i)) // 0..N contiguous, no duplicates
  899. const turnStarts = a2.session.snapshotEvents().filter(e => e.type === 'turn/start')
  900. expect(turnStarts.map(e => e.type === 'turn/start' && e.data.turn)).toEqual([1, 2])
  901. await ctx2.fiber.dispose()
  902. })
  903. it('resume rejects when session persistence is not configured', async () => {
  904. // A harness WITHOUT the persistence plugin.
  905. const adapter = new MockAdapter([textResponse('x')])
  906. const ctx = new Context()
  907. await ctx.plugin(LlmRuntime)
  908. await ctx.plugin(SessionStore)
  909. await ctx.plugin(SessionProjectionRegistry)
  910. await ctx.plugin(SystemPrompt)
  911. await ctx.plugin(ToolRuntime)
  912. await ctx.plugin(AgentRegistry)
  913. await ctx.plugin(AgentLoop, { agents: [] })
  914. ctx.llm.registerAdapter(['mock'], adapter)
  915. await expect(ctx.agents.resume({ resumeSessionId: SessionId('nope') }))
  916. .rejects.toThrow(/session persistence is not configured/)
  917. await ctx.fiber.dispose()
  918. })
  919. })
  920. describe('creation and resume cancellation edges', () => {
  921. it('rejects create() with a pre-aborted signal, including a non-Error reason', async () => {
  922. const { ctx } = await persistentHarness(new MockAdapter([]))
  923. const errorReason = new AbortController()
  924. errorReason.abort(new Error('caller gave up'))
  925. await expect(promptly(ctx.agents.create({
  926. sessionId: SessionId('pre-aborted-error'),
  927. agentOptions: { provider: 'mock', model: 'mock' },
  928. signal: errorReason.signal,
  929. }))).rejects.toThrow('caller gave up')
  930. // A non-Error reason is wrapped into the creation-aborted error.
  931. const stringReason = new AbortController()
  932. stringReason.abort('operator string reason')
  933. await expect(promptly(ctx.agents.create({
  934. sessionId: SessionId('pre-aborted-string'),
  935. agentOptions: { provider: 'mock', model: 'mock' },
  936. signal: stringReason.signal,
  937. }))).rejects.toThrow(/creation aborted/)
  938. expect(ctx.agents.get(SessionId('pre-aborted-error'))).toBeUndefined()
  939. expect(ctx.agents.get(SessionId('pre-aborted-string'))).toBeUndefined()
  940. await ctx.fiber.dispose()
  941. })
  942. it('a non-Error abort reason arriving during setup is wrapped for the caller', async () => {
  943. const { ctx } = await persistentHarness(new MockAdapter([]))
  944. const controller = new AbortController()
  945. const setupEntered = Promise.withResolvers<undefined>()
  946. const setupGate = Promise.withResolvers<undefined>()
  947. const creating = ctx.agents.create({
  948. sessionId: SessionId('setup-string-abort'),
  949. agentOptions: { provider: 'mock', model: 'mock' },
  950. signal: controller.signal,
  951. async setup() {
  952. setupEntered.resolve(undefined)
  953. await setupGate.promise
  954. },
  955. })
  956. await setupEntered.promise
  957. controller.abort('mid-setup string reason')
  958. setupGate.resolve(undefined)
  959. await expect(promptly(creating)).rejects.toThrow(/creation aborted/)
  960. expect(ctx.agents.get(SessionId('setup-string-abort'))).toBeUndefined()
  961. await ctx.fiber.dispose()
  962. })
  963. it('rejects when setup synchronously aborts its caller signal', async () => {
  964. const { ctx } = await persistentHarness(new MockAdapter([]))
  965. const controller = new AbortController()
  966. const creating = ctx.agents.create({
  967. sessionId: SessionId('setup-synchronous-abort'),
  968. agentOptions: { provider: 'mock', model: 'mock' },
  969. signal: controller.signal,
  970. setup() {
  971. controller.abort(new Error('setup synchronously cancelled'))
  972. },
  973. })
  974. await expect(promptly(creating)).rejects.toThrow('setup synchronously cancelled')
  975. expect(ctx.agents.get(SessionId('setup-synchronous-abort'))).toBeUndefined()
  976. await ctx.fiber.dispose()
  977. })
  978. it('resume with a pre-aborted caller signal rejects out of the load race', async () => {
  979. const sessionId = SessionId('resume-pre-aborted')
  980. const root = await persistSession(sessionId)
  981. const ctx = await mountPersistentHarness(root, new MockAdapter([]))
  982. const controller = new AbortController()
  983. controller.abort(new Error('resume abandoned'))
  984. await expect(promptly(ctx.agents.resume({
  985. resumeSessionId: sessionId,
  986. agentOptions: { provider: 'mock', model: 'mock' },
  987. signal: controller.signal,
  988. }))).rejects.toThrow('resume abandoned')
  989. const stringReason = new AbortController()
  990. stringReason.abort('resume string reason')
  991. await expect(promptly(ctx.agents.resume({
  992. resumeSessionId: sessionId,
  993. agentOptions: { provider: 'mock', model: 'mock' },
  994. signal: stringReason.signal,
  995. }))).rejects.toThrow(/creation aborted/)
  996. expect(ctx.agents.get(sessionId)).toBeUndefined()
  997. await ctx.fiber.dispose()
  998. })
  999. it('an abort landing between crash repair and publication refuses resume, normalized', async () => {
  1000. // Cover the publication-time abort backstop: the signal fires AFTER the
  1001. // raced open settled (during the closer append), so only the final
  1002. // pre-publication check can refuse.
  1003. const run = async (suffix: string, reason: unknown): Promise<unknown> => {
  1004. const sessionId = SessionId(`late-abort-${suffix}`)
  1005. const { ctx: seedCtx, root } = await persistentHarness(new MockAdapter([]))
  1006. await seedStoredSession(seedCtx, sessionId, [
  1007. { type: 'turn/start', seq: SessionSeq(0), time: 1, data: { turn: 1 } },
  1008. ])
  1009. await seedCtx.fiber.dispose()
  1010. const ctx = await mountPersistentHarness(root, new MockAdapter([]))
  1011. const controller = new AbortController()
  1012. const originalOpen = ctx.sessionPersistence.open.bind(ctx.sessionPersistence)
  1013. ctx.sessionPersistence.open = async (id, access, options) => {
  1014. const handle = await originalOpen(id, access, options)
  1015. const realAppend = handle.append.bind(handle)
  1016. Object.defineProperty(handle, 'append', {
  1017. value: (events: readonly SessionEvent[]) => {
  1018. controller.abort(reason)
  1019. return realAppend(events)
  1020. },
  1021. })
  1022. return handle
  1023. }
  1024. const rejection = await ctx.agents.resume({
  1025. resumeSessionId: sessionId,
  1026. agentOptions: { provider: 'mock', model: 'mock' },
  1027. signal: controller.signal,
  1028. }).then(() => undefined, (error: unknown) => error)
  1029. expect(ctx.agents.get(sessionId)).toBeUndefined()
  1030. await ctx.fiber.dispose()
  1031. return rejection
  1032. }
  1033. expect(await run('error', new Error('late abort error'))).toMatchObject({ message: 'late abort error' })
  1034. expect(await run('string', 'late abort string')).toMatchObject({ message: expect.stringMatching(/creation aborted/) as unknown })
  1035. })
  1036. it('an aborted create closes the write handle its abandoned backend create later resolves', async () => {
  1037. const sessionId = SessionId('create-abandoned-handle')
  1038. const { ctx } = await persistentHarness(new MockAdapter([]))
  1039. const lateCreate = Promise.withResolvers<SessionHandle>()
  1040. const createStarted = Promise.withResolvers<undefined>()
  1041. ctx.sessionPersistence.create = () => {
  1042. createStarted.resolve(undefined)
  1043. return lateCreate.promise
  1044. }
  1045. const controller = new AbortController()
  1046. const creating = ctx.agents.create({ sessionId, signal: controller.signal })
  1047. await createStarted.promise
  1048. controller.abort(new Error('caller aborted create'))
  1049. await expect(creating).rejects.toThrow('caller aborted create')
  1050. expect(ctx.sessions.get(sessionId)).toBeUndefined()
  1051. // The abandoned backend create still resolves a real write handle later;
  1052. // the loop closes it so ownership is not leaked, and a rejecting close is
  1053. // swallowed — there is no owner left to observe it.
  1054. const close = vi.fn(async () => { throw new Error('abandoned close failed') })
  1055. lateCreate.resolve({ append: async () => {}, close } as unknown as SessionHandle)
  1056. await expect.poll(() => close.mock.calls.length).toBe(1)
  1057. await ctx.fiber.dispose()
  1058. })
  1059. it('closes the resume write handle when the loop becomes inactive before setup', async () => {
  1060. const sessionId = SessionId('resume-loop-inactive-after-open')
  1061. const root = await persistSession(sessionId)
  1062. const ctx = await mountPersistentHarness(root, new MockAdapter([]))
  1063. const loop = ctx.agentLoop as unknown as {
  1064. ownership: { isActive: () => boolean }
  1065. }
  1066. vi.spyOn(loop.ownership, 'isActive').mockReturnValueOnce(false)
  1067. await expect(ctx.agents.resume({
  1068. resumeSessionId: sessionId,
  1069. agentOptions: { provider: 'mock', model: 'mock' },
  1070. })).rejects.toThrow('agent loop is not active')
  1071. expect(ctx.agents.get(sessionId)).toBeUndefined()
  1072. // The already-open handle was closed on the inactive path: a retry can
  1073. // claim write ownership again.
  1074. const retry = await ctx.agents.resume({
  1075. resumeSessionId: sessionId,
  1076. agentOptions: { provider: 'mock', model: 'mock' },
  1077. })
  1078. await retry.dispose()
  1079. await ctx.fiber.dispose()
  1080. })
  1081. it('factory teardown during a hung resume open rejects promptly and closes the late handle', async () => {
  1082. const sessionId = SessionId('resume-loop-teardown')
  1083. const root = await persistSession(sessionId)
  1084. const ctx = await mountPersistentHarness(root, new MockAdapter([]))
  1085. const gate = Promise.withResolvers<SessionHandle>()
  1086. const openStarted = Promise.withResolvers<undefined>()
  1087. ctx.sessionPersistence.open = () => {
  1088. openStarted.resolve(undefined)
  1089. return gate.promise
  1090. }
  1091. const resuming = ctx.agents.resume({
  1092. resumeSessionId: sessionId,
  1093. agentOptions: { provider: 'mock', model: 'mock' },
  1094. })
  1095. await openStarted.promise
  1096. // Resolve the open only after teardown began: the abandoned handle must
  1097. // still be released even though the wrapper already rejected.
  1098. const rejection = expect(promptly(resuming)).rejects.toThrow()
  1099. const disposal = ctx.fiber.dispose()
  1100. const abandoned = abandonedHandleStub()
  1101. gate.resolve(abandoned.handle)
  1102. await rejection
  1103. await disposal
  1104. await expect.poll(() => abandoned.close.mock.calls.length).toBe(1)
  1105. })
  1106. })
  1107. describe('configured-start failure edges', () => {
  1108. it('a non-Error mid-open abort reason is wrapped for the resume caller', async () => {
  1109. const sessionId = SessionId('resume-string-mid-abort')
  1110. const root = await persistSession(sessionId)
  1111. const ctx = await mountPersistentHarness(root, new MockAdapter([]))
  1112. const gate = Promise.withResolvers<never>()
  1113. gate.promise.catch(() => undefined)
  1114. const openStarted = Promise.withResolvers<undefined>()
  1115. ctx.sessionPersistence.open = () => {
  1116. openStarted.resolve(undefined)
  1117. return gate.promise
  1118. }
  1119. const controller = new AbortController()
  1120. const resuming = ctx.agents.resume({
  1121. resumeSessionId: sessionId,
  1122. agentOptions: { provider: 'mock', model: 'mock' },
  1123. signal: controller.signal,
  1124. })
  1125. await openStarted.promise
  1126. controller.abort('operator string reason')
  1127. await expect(promptly(resuming)).rejects.toThrow(/creation aborted/)
  1128. expect(ctx.agents.get(sessionId)).toBeUndefined()
  1129. await ctx.fiber.dispose()
  1130. })
  1131. it('a failing exact-id restore over an existing artifact stays loud', async () => {
  1132. const sessionId = SessionId('config-existing-corrupt')
  1133. const root = await persistSession(sessionId)
  1134. const configured = new Context()
  1135. await configured.plugin(LlmRuntime)
  1136. await configured.plugin(SessionStore)
  1137. await configured.plugin(SessionProjectionRegistry)
  1138. await configured.plugin(SystemPrompt)
  1139. await configured.plugin(ToolRuntime)
  1140. await configured.plugin(AgentRegistry)
  1141. await configured.plugin(JsonlSessionPersistence, { root })
  1142. configured.llm.registerAdapter(['mock'], new MockAdapter([]))
  1143. // The artifact exists but its open fails with a NON-NotFound error: this is
  1144. // corruption, not first creation — the failure must be reported, and no
  1145. // fresh same-id session may shadow the broken one.
  1146. configured.sessionPersistence.open = () => Promise.reject(new Error('artifact corrupt'))
  1147. const configFailures: unknown[] = []
  1148. configured.on('agent-loop/config-start-failed', ({ error }) => { configFailures.push(error) })
  1149. const warn = vi.spyOn(configured.logger, 'warn').mockImplementation(() => undefined)
  1150. const loop = await configured.plugin(AgentLoop, {
  1151. agents: [{ id: 'main', sessionId, provider: 'mock', model: 'mock' }],
  1152. })
  1153. await expect.poll(() => configFailures.length).toBe(1)
  1154. expect(configFailures[0]).toBeInstanceOf(Error)
  1155. expect((configFailures[0] as Error).message).toBe('artifact corrupt')
  1156. expect(warn).toHaveBeenCalledWith(expect.stringContaining('config-driven restore'))
  1157. expect(configured.agents.get(sessionId)).toBeUndefined()
  1158. warn.mockRestore()
  1159. await loop.dispose()
  1160. await configured.fiber.dispose()
  1161. })
  1162. it('suppresses a configured-resume failure that lands after teardown', async () => {
  1163. const sessionId = SessionId('config-late-resume-failure')
  1164. const root = await persistSession(sessionId)
  1165. const configured = new Context()
  1166. await configured.plugin(LlmRuntime)
  1167. await configured.plugin(SessionStore)
  1168. await configured.plugin(SessionProjectionRegistry)
  1169. await configured.plugin(SystemPrompt)
  1170. await configured.plugin(ToolRuntime)
  1171. await configured.plugin(AgentRegistry)
  1172. await configured.plugin(JsonlSessionPersistence, { root })
  1173. configured.llm.registerAdapter(['mock'], new MockAdapter([]))
  1174. const gate = Promise.withResolvers<never>()
  1175. gate.promise.catch(() => undefined)
  1176. const openStarted = Promise.withResolvers<undefined>()
  1177. configured.sessionPersistence.open = () => {
  1178. openStarted.resolve(undefined)
  1179. return gate.promise
  1180. }
  1181. const failures: unknown[] = []
  1182. configured.on('agent-loop/config-start-failed', ({ error }) => { failures.push(error) })
  1183. const loop = await configured.plugin(AgentLoop, {
  1184. agents: [{ id: 'main', resumeSessionId: sessionId, provider: 'mock', model: 'mock' }],
  1185. })
  1186. await openStarted.promise
  1187. const disposal = loop.dispose()
  1188. gate.reject(new Error('late backend failure'))
  1189. await disposal
  1190. await new Promise(r => setTimeout(r, 20))
  1191. // Ownership deactivated before the failure landed: the report is dropped.
  1192. expect(failures).toEqual([])
  1193. await configured.fiber.dispose()
  1194. })
  1195. })