resume.spec.ts 62 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340
  1. import { ToolCallId, createMessage, createUserMessage } from '@deepseek-ai/dsh-llm'
  2. import { afterEach, describe, expect, it, vi, type MockInstance } from 'vitest'
  3. import { Context } from '@deepseek-ai/cordis'
  4. import { appendFile, mkdtemp, readdir, rm } from 'node:fs/promises'
  5. import { tmpdir } from 'node:os'
  6. import { join } from 'node:path'
  7. import LlmRuntime from '@deepseek-ai/dsh-llm'
  8. import SessionStore, { SessionLogOffset, SessionSeq, Session, SessionId, TOOL_OUTCOME_UNKNOWN } from '@deepseek-ai/dsh-session'
  9. import type { SessionEvent } from '@deepseek-ai/dsh-session'
  10. import SystemPrompt from '@deepseek-ai/dsh-system-prompt'
  11. import ToolRuntime from '@deepseek-ai/dsh-tools'
  12. import AgentRegistry, { type Agent } from '@deepseek-ai/dsh-agent'
  13. import type { SessionHandle } from '@deepseek-ai/dsh-session-persistence'
  14. import JsonlSessionPersistence from '@deepseek-ai/dsh-session-persistence-jsonl'
  15. import AgentLoop from '@deepseek-ai/dsh-agent-loop'
  16. import SessionProjectionRegistry from '@deepseek-ai/dsh-session-projection'
  17. import { MockAdapter, textResponse } from './mock-adapter.ts'
  18. const dirs: string[] = []
  19. afterEach(async () => { for (const d of dirs.splice(0)) await rm(d, { recursive: true, force: true }) })
  20. async function persistentHarness(adapter: MockAdapter): Promise<{ ctx: Context; root: string }> {
  21. const root = await mkdtemp(join(tmpdir(), 'dsh-resume-'))
  22. dirs.push(root)
  23. return { ctx: await mountPersistentHarness(root, adapter), root }
  24. }
  25. async function mountPersistentHarness(root: string, adapter: MockAdapter, compression?: 'none'): Promise<Context> {
  26. const ctx = new Context()
  27. await ctx.plugin(LlmRuntime)
  28. await ctx.plugin(SessionStore)
  29. await ctx.plugin(SessionProjectionRegistry)
  30. await ctx.plugin(SystemPrompt)
  31. await ctx.plugin(ToolRuntime)
  32. await ctx.plugin(AgentRegistry)
  33. // The backend mounts BEFORE the loop so root teardown unwinds the loop
  34. // first: live agents drain their writers into still-open handles.
  35. await ctx.plugin(JsonlSessionPersistence, { root, ...compression === undefined ? {} : { compression } })
  36. await ctx.plugin(AgentLoop, { agents: [] })
  37. ctx.llm.registerAdapter(['mock'], adapter)
  38. return ctx
  39. }
  40. /** Remove every `session.lock` under the root: the POSIX forfeit-by-unlink escape hatch, without importing backend internals. */
  41. async function removeSessionLocks(dir: string): Promise<void> {
  42. for (const entry of await readdir(dir, { withFileTypes: true })) {
  43. const path = join(dir, entry.name)
  44. if (entry.isDirectory()) await removeSessionLocks(path)
  45. else if (entry.name === 'session.lock') await rm(path, { force: true })
  46. }
  47. }
  48. /** Seed one stored session through the persistence seam (header minted by the store). */
  49. async function seedStoredSession(ctx: Context, sessionId: SessionId, events: readonly SessionEvent[]): Promise<void> {
  50. const detached = ctx.sessions.prepare(sessionId)
  51. const handle = await ctx.sessionPersistence.create(detached.header)
  52. await handle.append(events)
  53. await handle.close()
  54. }
  55. /** Read one stored session's physical validated log through a read handle. */
  56. async function readStoredEvents(ctx: Context, sessionId: SessionId): Promise<readonly SessionEvent[]> {
  57. const handle = await ctx.sessionPersistence.open(sessionId, 'read')
  58. try {
  59. return (await handle.read()).events
  60. } finally {
  61. await handle.close()
  62. }
  63. }
  64. async function persistSession(sessionId: SessionId): Promise<string> {
  65. const { ctx, root } = await persistentHarness(new MockAdapter([]))
  66. // Persistence deliberately has no artifact for a truly empty session. A
  67. // balanced completed turn is the smallest resumable log and avoids running
  68. // the model merely to construct this lifecycle fixture.
  69. const seed: SessionEvent[] = [
  70. { type: 'turn/start', seq: SessionSeq(0), time: 1, data: { turn: 1 } },
  71. { type: 'turn/end', seq: SessionSeq(1), time: 2, data: { turn: 1, reason: { kind: 'completed' } } },
  72. ]
  73. await seedStoredSession(ctx, sessionId, seed)
  74. await ctx.fiber.dispose()
  75. return root
  76. }
  77. /** A handle stand-in for abandoned-open races; only `close()` is ever reachable. */
  78. function abandonedHandleStub(): { handle: SessionHandle; close: ReturnType<typeof vi.fn> } {
  79. const close = vi.fn(async () => {})
  80. return { handle: { close } as unknown as SessionHandle, close }
  81. }
  82. function waitForIdle(ctx: Context, agent: Agent): Promise<void> {
  83. return new Promise((resolve) => {
  84. const dispose = ctx.on('agent/status', ({ agent: subject, status }) => {
  85. if (subject === agent && status === 'idle') { dispose(); resolve() }
  86. })
  87. })
  88. }
  89. /** Fail a lifecycle regression promptly instead of waiting for Vitest's suite timeout. */
  90. async function promptly<T>(job: Promise<T>): Promise<T> {
  91. const timeout = Promise.withResolvers<never>()
  92. const timer = setTimeout(() => { timeout.reject(new Error('lifecycle task did not settle promptly')) }, 1000)
  93. try {
  94. return await Promise.race([job, timeout.promise])
  95. } finally {
  96. clearTimeout(timer)
  97. }
  98. }
  99. /** Throw an arbitrary callback value to exercise the public unknown-error boundary. */
  100. function throwUnknown(value: unknown): never {
  101. throw value
  102. }
  103. describe('the session-persistence Agent Note: AgentLoop factory create/resume', () => {
  104. it('normalizes a non-Error resume publication failure for rollback and releases the write handle', async () => {
  105. const sessionId = SessionId('unknown-resume-failure-s')
  106. const root = await persistSession(sessionId)
  107. const ctx = await mountPersistentHarness(root, new MockAdapter([textResponse('next')]))
  108. const failure = { source: 'resume' }
  109. ctx.on('session/created', () => throwUnknown(failure))
  110. await expect(ctx.agents.resume({
  111. resumeSessionId: sessionId,
  112. })).rejects.toBe(failure)
  113. expect(ctx.agents.get(SessionId('unknown-resume-failure'))).toBeUndefined()
  114. expect(ctx.sessions.get(sessionId)).toBeUndefined()
  115. // Rollback closed the write handle: write ownership is claimable again.
  116. const reopened = await ctx.sessionPersistence.open(sessionId, 'write')
  117. await reopened.close()
  118. await ctx.fiber.dispose()
  119. })
  120. it('createAgent uses the caller-supplied sessionId (not ${id}-session)', async () => {
  121. const adapter = new MockAdapter([textResponse('hi')])
  122. const { ctx } = await persistentHarness(adapter)
  123. const { agent } = await ctx.agents.create({ sessionId: SessionId('custom-session'), meta: { cwd: '/w' } })
  124. expect(agent.session.id).toBe('custom-session')
  125. expect(agent.session.header.cwd).toBe('/w')
  126. await ctx.fiber.dispose()
  127. })
  128. it('createAgent rejects a duplicate identity without orphaning a session', async () => {
  129. const adapter = new MockAdapter([textResponse('hi')])
  130. const { ctx } = await persistentHarness(adapter)
  131. const sessionId = SessionId('sess-a')
  132. await ctx.agents.create({ sessionId })
  133. await expect(ctx.agents.create({ sessionId })).rejects.toThrow(/already exists/)
  134. expect(ctx.sessions.list()).toHaveLength(1)
  135. await ctx.fiber.dispose()
  136. })
  137. it('a created agent stores its seed and live turn durably through its handle', async () => {
  138. const seed: SessionEvent[] = [
  139. { type: 'turn/start', seq: SessionSeq(0), time: 1, data: { turn: 1 } },
  140. { type: 'turn/end', seq: SessionSeq(1), time: 2, data: { turn: 1, reason: { kind: 'completed' } } },
  141. ]
  142. const { ctx } = await persistentHarness(new MockAdapter([textResponse('stored')]))
  143. const sessionId = SessionId('durable-create')
  144. const handle = await ctx.agents.create({
  145. sessionId,
  146. seed,
  147. meta: { cwd: '/w' },
  148. agentOptions: { provider: 'mock', model: 'mock' },
  149. })
  150. handle.agent.followup(createUserMessage({ content: [{ type: 'text', text: 'q' }], source: { kind: 'user' } }))
  151. await waitForIdle(ctx, handle.agent)
  152. await handle.dispose()
  153. const stored = await readStoredEvents(ctx, sessionId)
  154. const seqs = stored.map(event => event.seq)
  155. expect(seqs).toEqual(seqs.map((_, index) => index))
  156. // The constructor seed (turn 1 + its end-seed marker) precedes the live turn 2.
  157. expect(stored.slice(0, 2).map(event => event.type)).toEqual(['turn/start', 'turn/end'])
  158. expect(stored[2]?.type).toBe('session/end-seed')
  159. const turnStarts = stored.filter(event => event.type === 'turn/start')
  160. expect(turnStarts.map(event => event.type === 'turn/start' && event.data.turn)).toEqual([1, 2])
  161. expect(stored.some(event => event.type === 'turn/end' && event.data.turn === 2)).toBe(true)
  162. expect(JSON.stringify(stored)).toContain('stored')
  163. await ctx.fiber.dispose()
  164. })
  165. it('a rejecting final writer close releases the registries, then rejects disposal', async () => {
  166. const { ctx } = await persistentHarness(new MockAdapter([textResponse('hi')]))
  167. const sessionId = SessionId('drain-close-fails')
  168. const handle = await ctx.agents.create({ sessionId })
  169. const persisted = await ctx.sessionPersistence.open(sessionId, 'read')
  170. await persisted.close()
  171. const stored = [...(ctx.sessionPersistence as unknown as {
  172. tracker: { openHandles: Set<SessionHandle> }
  173. }).tracker.openHandles].find(open => open.id === sessionId && open.access === 'write')
  174. if (stored === undefined) throw new Error('missing owned write handle')
  175. // The real close still runs (releasing write ownership); the injected
  176. // failure models a drain that reports a durability error at close.
  177. const realClose = stored.close.bind(stored)
  178. vi.spyOn(stored, 'close').mockImplementation(async () => {
  179. await realClose()
  180. throw new Error('close exploded')
  181. })
  182. await expect(handle.dispose()).rejects.toThrow('close exploded')
  183. // Teardown reached quiescence before the rejection: the agent and session
  184. // are unregistered, and write ownership is released — the never-flushed
  185. // session reports absence, not an ownership conflict.
  186. expect(ctx.agents.get(sessionId)).toBeUndefined()
  187. expect(ctx.sessions.get(sessionId)).toBeUndefined()
  188. await expect(ctx.sessionPersistence.open(sessionId, 'write')).rejects.toThrow('not found')
  189. await ctx.fiber.dispose()
  190. })
  191. it('combines a machine-teardown failure with a close failure into one rejection', async () => {
  192. const { ctx } = await persistentHarness(new MockAdapter([textResponse('hi')]))
  193. const sessionId = SessionId('drain-both-fail')
  194. const handle = await ctx.agents.create({ sessionId })
  195. const stored = [...(ctx.sessionPersistence as unknown as {
  196. tracker: { openHandles: Set<SessionHandle> }
  197. }).tracker.openHandles].find(open => open.id === sessionId && open.access === 'write')
  198. if (stored === undefined) throw new Error('missing owned write handle')
  199. const machine = handle.agent as Agent & { scope: { dispose: () => Promise<void> } }
  200. vi.spyOn(machine.scope, 'dispose').mockRejectedValue(new Error('scope exploded'))
  201. vi.spyOn(stored, 'close').mockRejectedValue(new Error('close exploded'))
  202. const failure = await handle.dispose().then(() => undefined, (error: unknown) => error)
  203. if (!(failure instanceof AggregateError)) throw new Error('expected an AggregateError rejection')
  204. expect(failure.message).toContain(`agent "${sessionId}" disposal failed`)
  205. expect(failure.errors.map(error => (error as Error).message)).toEqual(['scope exploded', 'close exploded'])
  206. expect(ctx.agents.get(sessionId)).toBeUndefined()
  207. await ctx.fiber.dispose()
  208. })
  209. it('agent dispose releases write ownership of its stored session', async () => {
  210. const { ctx } = await persistentHarness(new MockAdapter([textResponse('hi')]))
  211. const sessionId = SessionId('ownership-release')
  212. const handle = await ctx.agents.create({ sessionId })
  213. await expect(ctx.sessionPersistence.open(sessionId, 'write'))
  214. .rejects.toThrow(/already owned by an active write handle/)
  215. // Materialize the log so the session outlives its creator handle.
  216. handle.agent.followup(createUserMessage({ content: [{ type: 'text', text: 'q' }], source: { kind: 'user' } }))
  217. await waitForIdle(ctx, handle.agent)
  218. await handle.dispose()
  219. const reopened = await ctx.sessionPersistence.open(sessionId, 'write')
  220. expect(reopened.access).toBe('write')
  221. await reopened.close()
  222. await ctx.fiber.dispose()
  223. })
  224. it('a stored-session create failure rolls the fresh identity back', async () => {
  225. const { ctx } = await persistentHarness(new MockAdapter([]))
  226. const sessionId = SessionId('create-backend-fail')
  227. vi.spyOn(ctx.sessionPersistence, 'create').mockRejectedValueOnce(new Error('backend create failed'))
  228. await expect(ctx.agents.create({ sessionId })).rejects.toThrow('backend create failed')
  229. expect(ctx.agents.get(sessionId)).toBeUndefined()
  230. expect(ctx.sessions.get(sessionId)).toBeUndefined()
  231. // The identity was fully released: the same id creates cleanly afterwards.
  232. const retry = await ctx.agents.create({ sessionId })
  233. await retry.dispose()
  234. await ctx.fiber.dispose()
  235. })
  236. it('a seed append failure closes the fresh write handle and rethrows', async () => {
  237. const { ctx } = await persistentHarness(new MockAdapter([]))
  238. const sessionId = SessionId('seed-append-fail')
  239. const seed: SessionEvent[] = [
  240. { type: 'turn/start', seq: SessionSeq(0), time: 1, data: { turn: 1 } },
  241. { type: 'turn/end', seq: SessionSeq(1), time: 2, data: { turn: 1, reason: { kind: 'completed' } } },
  242. ]
  243. const originalCreate = ctx.sessionPersistence.create.bind(ctx.sessionPersistence)
  244. let closeSpy: MockInstance<() => Promise<void>> | undefined
  245. ctx.sessionPersistence.create = async (header, options) => {
  246. const handle = await originalCreate(header, options)
  247. vi.spyOn(handle, 'append').mockRejectedValue(new Error('seed append failed'))
  248. closeSpy = vi.spyOn(handle, 'close')
  249. return handle
  250. }
  251. await expect(ctx.agents.create({ sessionId, seed })).rejects.toThrow('seed append failed')
  252. expect(closeSpy).toHaveBeenCalled()
  253. expect(ctx.agents.get(sessionId)).toBeUndefined()
  254. expect(ctx.sessions.get(sessionId)).toBeUndefined()
  255. // The closed never-materialized creation left no stored session behind.
  256. await expect(ctx.sessionPersistence.open(sessionId, 'write'))
  257. .rejects.toThrow(/not found/)
  258. await ctx.fiber.dispose()
  259. })
  260. it('a setup failure before publication leaves no stored residue; the id creates again', async () => {
  261. const { ctx } = await persistentHarness(new MockAdapter([]))
  262. const sessionId = SessionId('setup-fail-no-residue')
  263. const seed: SessionEvent[] = [
  264. { type: 'turn/start', seq: SessionSeq(0), time: 1, data: { turn: 1 } },
  265. { type: 'turn/end', seq: SessionSeq(1), time: 2, data: { turn: 1, reason: { kind: 'completed' } } },
  266. ]
  267. await expect(ctx.agents.create({
  268. sessionId,
  269. seed,
  270. setup: () => { throw new Error('setup refused') },
  271. })).rejects.toThrow('setup refused')
  272. // The seed is stored only at the publication commit point, so the failed
  273. // attempt materialized nothing and released the identity completely.
  274. await expect(ctx.sessionPersistence.stat(sessionId)).resolves.toBeUndefined()
  275. const retry = await ctx.agents.create({ sessionId, seed })
  276. await expect(ctx.sessionPersistence.stat(sessionId)).resolves.toBeDefined()
  277. await retry.dispose()
  278. await ctx.fiber.dispose()
  279. })
  280. it('rollback swallows a rejecting handle close after a prepare failure (create and createAgent)', async () => {
  281. const { ctx } = await persistentHarness(new MockAdapter([]))
  282. const originalCreate = ctx.sessionPersistence.create.bind(ctx.sessionPersistence)
  283. const closeSpies: Array<{ mockRestore: () => void }> = []
  284. ctx.sessionPersistence.create = async (header, options) => {
  285. const handle = await originalCreate(header, options)
  286. closeSpies.push(vi.spyOn(handle, 'close').mockRejectedValue(new Error('close failed')))
  287. return handle
  288. }
  289. // Config create path: prepare's option validation throws after the handle exists.
  290. await expect(ctx.agentLoop.create(SessionId('close-reject-config'), { maxTokens: -1 }))
  291. .rejects.toThrow('agent maxTokens must be a positive safe integer')
  292. expect(ctx.agents.get(SessionId('close-reject-config'))).toBeUndefined()
  293. // Owned createAgent path: the same validation failure after the handle exists.
  294. await expect(ctx.agents.create({
  295. sessionId: SessionId('close-reject-owned'),
  296. agentOptions: { maxTokens: -1 },
  297. })).rejects.toThrow('agent maxTokens must be a positive safe integer')
  298. expect(ctx.agents.get(SessionId('close-reject-owned'))).toBeUndefined()
  299. for (const spy of closeSpies.splice(0)) spy.mockRestore()
  300. await ctx.fiber.dispose()
  301. })
  302. it('a reentrant abort during preparation swallows a rejecting handle close', async () => {
  303. const { ctx } = await persistentHarness(new MockAdapter([]))
  304. const sessionId = SessionId('prepare-abort-close-reject')
  305. const originalCreate = ctx.sessionPersistence.create.bind(ctx.sessionPersistence)
  306. const spies: Array<{ mockRestore: () => void }> = []
  307. let closed: Promise<void> | undefined
  308. ctx.sessionPersistence.create = async (header, options) => {
  309. const handle = await originalCreate(header, options)
  310. const realClose = handle.close.bind(handle)
  311. spies.push(vi.spyOn(handle, 'close').mockImplementation(async () => {
  312. closed = realClose()
  313. await closed
  314. throw new Error('close failed')
  315. }))
  316. return handle
  317. }
  318. const reason = new Error('cancelled while preparing')
  319. const controller = new AbortController()
  320. let aborted = false
  321. ctx.on('internal/plugin', (fiber) => {
  322. if (aborted || fiber.name !== 'scope') return
  323. aborted = true
  324. controller.abort(reason)
  325. })
  326. await expect(ctx.agents.create({ sessionId, signal: controller.signal })).rejects.toBe(reason)
  327. // The rejecting close stays swallowed by the rollback; wait for the
  328. // rollback's real close so factory teardown finds a quiescent lifecycle.
  329. await vi.waitFor(() => { if (closed === undefined) throw new Error('close not reached') })
  330. await closed
  331. expect(ctx.agents.get(sessionId)).toBeUndefined()
  332. for (const spy of spies.splice(0)) spy.mockRestore()
  333. await ctx.fiber.dispose()
  334. })
  335. it('config create rollback swallows a rejecting close after a publish failure', async () => {
  336. const { ctx } = await persistentHarness(new MockAdapter([]))
  337. const sessionId = SessionId('config-publish-close-reject')
  338. const originalCreate = ctx.sessionPersistence.create.bind(ctx.sessionPersistence)
  339. const spies: Array<{ mockRestore: () => void }> = []
  340. let closed: Promise<void> | undefined
  341. ctx.sessionPersistence.create = async (header, options) => {
  342. const handle = await originalCreate(header, options)
  343. const realClose = handle.close.bind(handle)
  344. spies.push(vi.spyOn(handle, 'close').mockImplementation(async () => {
  345. closed = realClose()
  346. await closed
  347. throw new Error('close failed')
  348. }))
  349. return handle
  350. }
  351. const announce = vi.spyOn(ctx.agents, 'announce').mockImplementation(() => {
  352. throw new Error('announce failed')
  353. })
  354. await expect(ctx.agentLoop.create(sessionId)).rejects.toThrow('announce failed')
  355. announce.mockRestore()
  356. await vi.waitFor(() => { if (closed === undefined) throw new Error('close not reached') })
  357. await closed
  358. expect(ctx.agents.get(sessionId)).toBeUndefined()
  359. for (const spy of spies.splice(0)) spy.mockRestore()
  360. await ctx.fiber.dispose()
  361. })
  362. it('a seed append failure swallows a rejecting handle close', async () => {
  363. const { ctx } = await persistentHarness(new MockAdapter([]))
  364. const sessionId = SessionId('seed-append-close-reject')
  365. const seed: SessionEvent[] = [
  366. { type: 'turn/start', seq: SessionSeq(0), time: 1, data: { turn: 1 } },
  367. { type: 'turn/end', seq: SessionSeq(1), time: 2, data: { turn: 1, reason: { kind: 'completed' } } },
  368. ]
  369. const originalCreate = ctx.sessionPersistence.create.bind(ctx.sessionPersistence)
  370. const spies: Array<{ mockRestore: () => void }> = []
  371. ctx.sessionPersistence.create = async (header, options) => {
  372. const handle = await originalCreate(header, options)
  373. spies.push(vi.spyOn(handle, 'append').mockRejectedValue(new Error('seed append failed')))
  374. spies.push(vi.spyOn(handle, 'close').mockRejectedValue(new Error('close failed')))
  375. return handle
  376. }
  377. await expect(ctx.agents.create({ sessionId, seed })).rejects.toThrow('seed append failed')
  378. expect(ctx.agents.get(sessionId)).toBeUndefined()
  379. expect(ctx.sessions.get(sessionId)).toBeUndefined()
  380. for (const spy of spies.splice(0)) spy.mockRestore()
  381. await ctx.fiber.dispose()
  382. })
  383. it('a resume read failure swallows a rejecting handle close during rollback', async () => {
  384. const sessionId = SessionId('resume-read-close-reject')
  385. const root = await persistSession(sessionId)
  386. const ctx = await mountPersistentHarness(root, new MockAdapter([]))
  387. const originalOpen = ctx.sessionPersistence.open.bind(ctx.sessionPersistence)
  388. const spies: Array<{ mockRestore: () => void }> = []
  389. ctx.sessionPersistence.open = async (id, access, options) => {
  390. const handle = await originalOpen(id, access, options)
  391. spies.push(vi.spyOn(handle, 'read').mockRejectedValue(new Error('stored read failed')))
  392. spies.push(vi.spyOn(handle, 'close').mockRejectedValue(new Error('close failed')))
  393. return handle
  394. }
  395. await expect(ctx.agents.resume({ resumeSessionId: sessionId }))
  396. .rejects.toThrow('stored read failed')
  397. expect(ctx.agents.get(sessionId)).toBeUndefined()
  398. expect(ctx.sessions.get(sessionId)).toBeUndefined()
  399. for (const spy of spies.splice(0)) spy.mockRestore()
  400. await ctx.fiber.dispose()
  401. })
  402. it('resume cannot crash-repair a turn owned by a live agent', async () => {
  403. const { ctx } = await persistentHarness(new MockAdapter([textResponse('unused')]))
  404. const sessionId = SessionId('live-resume-race')
  405. const first = (await ctx.agents.create({ sessionId })).agent
  406. first.session.append('turn/start', { turn: 1 })
  407. await ctx.sessions.flush(first.session)
  408. await expect(ctx.agents.resume({ resumeSessionId: sessionId }))
  409. .rejects.toThrow(/already owned by an active write handle/)
  410. first.session.append('turn/end', { turn: 1, reason: { kind: 'completed' } })
  411. await ctx.sessions.flush(first.session)
  412. const stored = await readStoredEvents(ctx, sessionId)
  413. expect(stored.map(event => event.type)).toEqual(['turn/start', 'turn/end'])
  414. expect(stored.at(-1)).toMatchObject({
  415. type: 'turn/end',
  416. data: { reason: { kind: 'completed' } },
  417. })
  418. await ctx.fiber.dispose()
  419. })
  420. it('resume appends interrupted-turn closers durably through the handle', async () => {
  421. // Lifecycle 1: store an interrupted log — an open turn with no turn/end.
  422. const sessionId = SessionId('interrupted-resume')
  423. const { ctx: ctx1, root } = await persistentHarness(new MockAdapter([]))
  424. await seedStoredSession(ctx1, sessionId, [
  425. { type: 'turn/start', seq: SessionSeq(0), time: 1, data: { turn: 1 } },
  426. ])
  427. // A read returns the PHYSICAL validated log: no synthetic closers.
  428. const raw = await readStoredEvents(ctx1, sessionId)
  429. expect(raw.map(event => event.type)).toEqual(['turn/start'])
  430. await ctx1.fiber.dispose()
  431. // Lifecycle 2: resume repairs the tail and stores the repair durably.
  432. const ctx2 = await mountPersistentHarness(root, new MockAdapter([]))
  433. const handle = await ctx2.agents.resume({ resumeSessionId: sessionId })
  434. expect(handle.agent.session.snapshotEvents().map(event => event.type))
  435. .toEqual(['turn/start', 'turn/end', 'session/end-seed'])
  436. await handle.dispose()
  437. const stored = await readStoredEvents(ctx2, sessionId)
  438. expect(stored.map(event => event.type)).toEqual(['turn/start', 'turn/end', 'session/end-seed'])
  439. expect(stored[1]).toMatchObject({ data: { reason: { kind: 'interrupted' } } })
  440. await ctx2.fiber.dispose()
  441. })
  442. it('resume closes an interrupted tool call durably: tool/result, step/end, turn/end', async () => {
  443. const sessionId = SessionId('interrupted-tool-resume')
  444. const { ctx: ctx1, root } = await persistentHarness(new MockAdapter([]))
  445. await seedStoredSession(ctx1, sessionId, [
  446. { type: 'turn/start', seq: SessionSeq(0), time: 1, data: { turn: 1 } },
  447. { type: 'step/start', seq: SessionSeq(1), time: 1, data: { turn: 1, step: 1 } },
  448. { type: 'assistant/message', seq: SessionSeq(2), time: 2, surfaceOp: 'append', data: {
  449. turn: 1, step: 1,
  450. stream: [],
  451. message: createMessage({
  452. role: 'assistant',
  453. content: [{ type: 'tool-call', id: ToolCallId('call-1'), name: 'bash', arguments: '{}' }],
  454. source: { kind: 'model', provider: 'mock', model: 'mock' },
  455. }),
  456. } },
  457. { type: 'tool/call', seq: SessionSeq(3), time: 2, data: { turn: 1, step: 1, callId: ToolCallId('call-1'), name: 'bash', arguments: '{}' } },
  458. ] as SessionEvent[])
  459. await ctx1.fiber.dispose()
  460. const ctx2 = await mountPersistentHarness(root, new MockAdapter([]))
  461. const handle = await ctx2.agents.resume({ resumeSessionId: sessionId })
  462. await handle.dispose()
  463. // The multi-closer set lands durably in one contiguous batch, and the
  464. // synthetic tool/result cites the recorded tool/call seq.
  465. const stored = await readStoredEvents(ctx2, sessionId)
  466. expect(stored.map(event => event.type)).toEqual([
  467. 'turn/start', 'step/start', 'assistant/message', 'tool/call',
  468. 'tool/result', 'step/end', 'turn/end', 'session/end-seed',
  469. ])
  470. expect(stored.map(event => event.seq)).toEqual([0, 1, 2, 3, 4, 5, 6, 7])
  471. expect(stored[4]).toMatchObject({
  472. sourceEventSeqs: [3],
  473. data: { error: { code: TOOL_OUTCOME_UNKNOWN } },
  474. })
  475. expect(stored[6]).toMatchObject({ data: { reason: { kind: 'interrupted' } } })
  476. await ctx2.fiber.dispose()
  477. })
  478. it('resume over a torn physical tail continues from the committed prefix', async () => {
  479. const sessionId = SessionId('torn-tail-resume')
  480. const root = await mkdtemp(join(tmpdir(), 'dsh-resume-torn-'))
  481. dirs.push(root)
  482. const ctx1 = await mountPersistentHarness(root, new MockAdapter([]), 'none')
  483. await seedStoredSession(ctx1, sessionId, [
  484. { type: 'turn/start', seq: SessionSeq(0), time: 1, data: { turn: 1 } },
  485. ])
  486. await ctx1.fiber.dispose()
  487. // Crash artifact: a half-written record with no trailing newline.
  488. const logs = (await readdir(root, { recursive: true })).filter(name => name.endsWith('.jsonl'))
  489. expect(logs).toHaveLength(1)
  490. await appendFile(join(root, logs[0] as string), '{"type":"assistant/chunk","seq":1,"ti')
  491. // Resume truncates the torn tail under its write open, then appends the
  492. // closers immediately after the committed prefix — no gap, no fragment.
  493. const ctx2 = await mountPersistentHarness(root, new MockAdapter([]), 'none')
  494. const handle = await ctx2.agents.resume({ resumeSessionId: sessionId })
  495. expect(handle.agent.session.snapshotEvents().map(event => event.type))
  496. .toEqual(['turn/start', 'turn/end', 'session/end-seed'])
  497. await handle.dispose()
  498. const stored = await readStoredEvents(ctx2, sessionId)
  499. expect(stored.map(event => `${event.type}@${event.seq}`))
  500. .toEqual(['turn/start@0', 'turn/end@1', 'session/end-seed@2'])
  501. await ctx2.fiber.dispose()
  502. })
  503. it('createAgent works without meta (no cwd)', async () => {
  504. const adapter = new MockAdapter([textResponse('hi')])
  505. const { ctx } = await persistentHarness(adapter)
  506. const { agent } = await ctx.agents.create({ sessionId: SessionId('nometa-session') })
  507. expect(agent.session.id).toBe('nometa-session')
  508. expect(agent.session.header.cwd).toBeUndefined()
  509. await ctx.fiber.dispose()
  510. })
  511. it('resume of a session with no cwd carries an undefined cwd header', async () => {
  512. // Lifecycle 1: create a no-cwd session and run a turn.
  513. const adapter1 = new MockAdapter([textResponse('a')])
  514. const { ctx: ctx1, root } = await persistentHarness(adapter1)
  515. const h1 = await ctx1.agents.create({ sessionId: SessionId('nocwd-sess') })
  516. h1.agent.followup(createUserMessage({ content: [{ type: 'text', text: 'q' }], source: { kind: 'user' } }))
  517. await waitForIdle(ctx1, h1.agent)
  518. // Agent disposal drains the writer through the still-open handle.
  519. await h1.dispose()
  520. await ctx1.fiber.dispose()
  521. // Lifecycle 2: resume it; the header cwd stays undefined (no-cwd branch).
  522. const ctx2 = await mountPersistentHarness(root, new MockAdapter([textResponse('b')]))
  523. const a2 = (await ctx2.agents.resume({ resumeSessionId: SessionId('nocwd-sess') })).agent
  524. expect(a2.session.header.cwd).toBeUndefined()
  525. await ctx2.fiber.dispose()
  526. })
  527. it('agent/session-start fires "startup" for createAgent and "resume" for resume()', async () => {
  528. // Lifecycle 1: a fresh createAgent emits session-start with source 'startup'.
  529. const adapter1 = new MockAdapter([textResponse('a')])
  530. const { ctx: ctx1, root } = await persistentHarness(adapter1)
  531. const sources1: string[] = []
  532. ctx1.on('agent/session-start', ({ source }) => void sources1.push(source))
  533. const h1 = await ctx1.agents.create({ sessionId: SessionId('start-sess') })
  534. expect(sources1).toEqual(['startup'])
  535. h1.agent.followup(createUserMessage({ content: [{ type: 'text', text: 'q' }], source: { kind: 'user' } }))
  536. await waitForIdle(ctx1, h1.agent)
  537. await h1.dispose()
  538. await ctx1.fiber.dispose()
  539. // Lifecycle 2: resuming the persisted session emits session-start 'resume'.
  540. const ctx2 = await mountPersistentHarness(root, new MockAdapter([textResponse('b')]))
  541. const sources2: string[] = []
  542. ctx2.on('agent/session-start', ({ source }) => void sources2.push(source))
  543. await ctx2.agents.resume({ resumeSessionId: SessionId('start-sess') })
  544. expect(sources2).toEqual(['resume'])
  545. await ctx2.fiber.dispose()
  546. })
  547. it('resume awaits setup while unpublished, then publishes a fully composed world in order', async () => {
  548. const sessionId = SessionId('resume-setup-success')
  549. const root = await persistSession(sessionId)
  550. const ctx = await mountPersistentHarness(root, new MockAdapter([textResponse('next')]))
  551. const gate = Promise.withResolvers<undefined>()
  552. const setupStarted = Promise.withResolvers<undefined>()
  553. const order: string[] = []
  554. ctx.on('session/created', (session) => {
  555. expect(ctx.sessions.get(session.id)).toBe(session)
  556. expect(ctx.agents.get(sessionId)?.session).toBe(session)
  557. order.push('session/created')
  558. })
  559. ctx.on('agent/created', ({ agent }) => {
  560. expect(agent.status).toBe('idle')
  561. order.push('agent/created')
  562. })
  563. ctx.on('agent/session-start', ({ agent }) => {
  564. expect(() => { agent.cancel({ kind: 'user' }) }).not.toThrow()
  565. order.push('agent/session-start')
  566. })
  567. const resuming = ctx.agents.resume({
  568. resumeSessionId: sessionId,
  569. agentOptions: { provider: 'mock', model: 'mock' },
  570. setup: async (agentCtx, agent) => {
  571. expect(agent.id).toBe(sessionId)
  572. // The two persisted events plus the end-seed marker.
  573. expect(agent.session.snapshotEvents()).toHaveLength(3)
  574. agentCtx.on('session/created', () => void order.push('setup-listener:session/created'))
  575. agentCtx.on('agent/created', () => void order.push('setup-listener:agent/created'))
  576. order.push('setup:start')
  577. setupStarted.resolve(undefined)
  578. await gate.promise
  579. order.push('setup:end')
  580. return {
  581. commit: () => {
  582. expect(ctx.agents.get(sessionId)).toBeUndefined()
  583. expect(ctx.sessions.get(sessionId)).toBeUndefined()
  584. order.push('setup:commit')
  585. },
  586. }
  587. },
  588. })
  589. await setupStarted.promise
  590. expect(ctx.agents.get(sessionId)).toBeUndefined()
  591. expect(ctx.sessions.get(sessionId)).toBeUndefined()
  592. expect(order).toEqual(['setup:start'])
  593. gate.resolve(undefined)
  594. const handle = await resuming
  595. expect(order).toEqual([
  596. 'setup:start',
  597. 'setup:end',
  598. 'setup:commit',
  599. 'session/created',
  600. 'setup-listener:session/created',
  601. 'agent/created',
  602. 'setup-listener:agent/created',
  603. 'agent/session-start',
  604. ])
  605. await handle.dispose()
  606. await ctx.fiber.dispose()
  607. })
  608. it('a resumed session stores its repair suffix durably before publication', async () => {
  609. // The stored fixture (two events, no end-seed) gains the end-seed marker
  610. // through the resume handle: after one resume lifecycle the STORED log
  611. // carries it, so the next resume reads it back without re-marking.
  612. const sessionId = SessionId('resume-suffix-durable')
  613. const root = await persistSession(sessionId)
  614. const ctx = await mountPersistentHarness(root, new MockAdapter([]))
  615. const first = await ctx.agents.resume({ resumeSessionId: sessionId })
  616. await first.dispose()
  617. const stored = await readStoredEvents(ctx, sessionId)
  618. expect(stored.map(event => event.type)).toEqual(['turn/start', 'turn/end', 'session/end-seed'])
  619. const second = await ctx.agents.resume({ resumeSessionId: sessionId })
  620. expect(second.agent.session.snapshotEvents().map(event => event.type))
  621. .toEqual(['turn/start', 'turn/end', 'session/end-seed'])
  622. await second.dispose()
  623. const restored = await readStoredEvents(ctx, sessionId)
  624. expect(restored.map(event => event.type)).toEqual(['turn/start', 'turn/end', 'session/end-seed'])
  625. await ctx.fiber.dispose()
  626. })
  627. it('successful resume disposal retires its caller-owned transaction effects', async () => {
  628. const sessionId = SessionId('resume-retired-effects-s')
  629. const root = await persistSession(sessionId)
  630. const ctx = await mountPersistentHarness(root, new MockAdapter([textResponse('next')]))
  631. const handle = await ctx.agents.resume({
  632. resumeSessionId: sessionId,
  633. agentOptions: { provider: 'mock', model: 'mock' },
  634. })
  635. const transactionLabels = [`agentLoop.lifecycle(${sessionId})`]
  636. expect(ctx.fiber.getEffects().map(effect => effect.label)).toEqual(expect.arrayContaining(transactionLabels))
  637. await handle.dispose()
  638. expect(ctx.fiber.getEffects().filter(effect => transactionLabels.includes(effect.label))).toEqual([])
  639. await ctx.fiber.dispose()
  640. })
  641. it('resume setup rejection publishes nothing, unwinds, and releases the identity', async () => {
  642. const sessionId = SessionId('resume-setup-reject')
  643. const root = await persistSession(sessionId)
  644. const ctx = await mountPersistentHarness(root, new MockAdapter([textResponse('next')]))
  645. const published: string[] = []
  646. ctx.on('session/created', () => void published.push('session/created'))
  647. ctx.on('agent/created', () => void published.push('agent/created'))
  648. ctx.on('agent/session-start', () => void published.push('agent/session-start'))
  649. await expect(ctx.agents.resume({
  650. resumeSessionId: sessionId,
  651. agentOptions: { provider: 'mock', model: 'mock' },
  652. setup: async () => {
  653. await Promise.resolve()
  654. throw new Error('resume setup failed')
  655. },
  656. })).rejects.toThrow('resume setup failed')
  657. expect(published).toEqual([])
  658. expect(ctx.agents.get(sessionId)).toBeUndefined()
  659. expect(ctx.sessions.get(sessionId)).toBeUndefined()
  660. const retry = await ctx.agents.resume({
  661. resumeSessionId: sessionId,
  662. agentOptions: { provider: 'mock', model: 'mock' },
  663. })
  664. await retry.dispose()
  665. await ctx.fiber.dispose()
  666. })
  667. it('resume setup commit rejection publishes nothing and releases the identity', async () => {
  668. const sessionId = SessionId('resume-setup-commit-reject')
  669. const root = await persistSession(sessionId)
  670. const ctx = await mountPersistentHarness(root, new MockAdapter([textResponse('next')]))
  671. const published: string[] = []
  672. ctx.on('session/created', () => void published.push('session/created'))
  673. ctx.on('agent/created', () => void published.push('agent/created'))
  674. await expect(ctx.agents.resume({
  675. resumeSessionId: sessionId,
  676. agentOptions: { provider: 'mock', model: 'mock' },
  677. setup: () => ({
  678. commit: () => { throw new Error('resume setup commit failed') },
  679. }),
  680. })).rejects.toThrow('resume setup commit failed')
  681. expect(published).toEqual([])
  682. expect(ctx.agents.get(sessionId)).toBeUndefined()
  683. expect(ctx.sessions.get(sessionId)).toBeUndefined()
  684. const retry = await ctx.agents.resume({
  685. resumeSessionId: sessionId,
  686. agentOptions: { provider: 'mock', model: 'mock' },
  687. })
  688. await retry.dispose()
  689. await ctx.fiber.dispose()
  690. })
  691. it('owner unload aborts resume setup and cannot publish after the callback settles', async () => {
  692. const sessionId = SessionId('resume-setup-owner-unload')
  693. const root = await persistSession(sessionId)
  694. const ctx = await mountPersistentHarness(root, new MockAdapter([textResponse('next')]))
  695. const gate = Promise.withResolvers<undefined>()
  696. const setupStarted = Promise.withResolvers<undefined>()
  697. const published: string[] = []
  698. ctx.on('session/created', () => void published.push('session/created'))
  699. ctx.on('agent/created', () => void published.push('agent/created'))
  700. let resuming!: ReturnType<typeof ctx.agents.resume>
  701. const owner = await ctx.plugin(Object.assign((inner: Context) => {
  702. resuming = inner.agents.resume({
  703. resumeSessionId: sessionId,
  704. agentOptions: { provider: 'mock', model: 'mock' },
  705. setup: async () => {
  706. setupStarted.resolve(undefined)
  707. await gate.promise
  708. },
  709. })
  710. }, { inject: ['agents'] }))
  711. await setupStarted.promise
  712. await owner.dispose()
  713. await expect(resuming).rejects.toThrow(/owner disposed during setup/)
  714. expect(published).toEqual([])
  715. expect(ctx.agents.get(sessionId)).toBeUndefined()
  716. expect(ctx.sessions.get(sessionId)).toBeUndefined()
  717. gate.resolve(undefined)
  718. await Promise.resolve()
  719. expect(published).toEqual([])
  720. await ctx.fiber.dispose()
  721. })
  722. it('owner unload aborts a never-settling persistence open, releases the identity, and blocks late publication', async () => {
  723. const sessionId = SessionId('resume-load-owner-unload')
  724. const root = await persistSession(sessionId)
  725. const ctx = await mountPersistentHarness(root, new MockAdapter([textResponse('next')]))
  726. const lateOpen = Promise.withResolvers<SessionHandle>()
  727. const openStarted = Promise.withResolvers<undefined>()
  728. const originalOpen = ctx.sessionPersistence.open.bind(ctx.sessionPersistence)
  729. let opens = 0
  730. ctx.sessionPersistence.open = (id, access, options) => {
  731. expect(id).toBe(sessionId)
  732. opens += 1
  733. if (opens === 1) {
  734. openStarted.resolve(undefined)
  735. return lateOpen.promise
  736. }
  737. return originalOpen(id, access, options)
  738. }
  739. const published: string[] = []
  740. ctx.on('session/created', () => void published.push('session/created'))
  741. ctx.on('agent/created', () => void published.push('agent/created'))
  742. ctx.on('agent/session-start', () => void published.push('agent/session-start'))
  743. let resuming!: ReturnType<typeof ctx.agents.resume>
  744. const owner = await ctx.plugin(Object.assign((inner: Context) => {
  745. resuming = inner.agents.resume({ resumeSessionId: sessionId, agentOptions: { provider: 'mock', model: 'mock' } })
  746. }, { inject: ['agents'] }))
  747. await openStarted.promise
  748. const rejection = expect(promptly(resuming)).rejects.toThrow(/owner disposed during setup/)
  749. await promptly(owner.dispose())
  750. expect(published).toEqual([])
  751. expect(ctx.agents.get(sessionId)).toBeUndefined()
  752. expect(ctx.sessions.get(sessionId)).toBeUndefined()
  753. // owner.dispose() awaited transaction settlement, so the same identities
  754. // can be reused before awaiting the public rejection.
  755. const retry = await promptly(ctx.agents.resume({ resumeSessionId: sessionId, agentOptions: { provider: 'mock', model: 'mock' } }))
  756. await rejection
  757. expect(opens).toBe(2)
  758. expect(published).toEqual(['session/created', 'agent/created', 'agent/session-start'])
  759. // Settlement of the abandoned backend open cannot resume the old
  760. // transaction: the late handle is closed, and no second publication lands
  761. // after the retry owns the ids.
  762. const abandoned = abandonedHandleStub()
  763. lateOpen.resolve(abandoned.handle)
  764. await expect.poll(() => abandoned.close.mock.calls.length).toBe(1)
  765. expect(ctx.agents.get(sessionId)).toBe(retry.agent)
  766. expect(ctx.sessions.get(sessionId)).toBe(retry.agent.session)
  767. expect(published).toEqual(['session/created', 'agent/created', 'agent/session-start'])
  768. await retry.dispose()
  769. await ctx.fiber.dispose()
  770. })
  771. it('AgentLoop unload aborts a never-settling persistence open and awaits wrapper settlement', async () => {
  772. const sessionId = SessionId('resume-load-factory-unload')
  773. const root = await persistSession(sessionId)
  774. const ctx = new Context()
  775. await ctx.plugin(LlmRuntime)
  776. await ctx.plugin(SessionStore)
  777. await ctx.plugin(SessionProjectionRegistry)
  778. await ctx.plugin(SystemPrompt)
  779. await ctx.plugin(ToolRuntime)
  780. await ctx.plugin(AgentRegistry)
  781. const loopFiber = await ctx.plugin(AgentLoop, { agents: [] })
  782. await ctx.plugin(JsonlSessionPersistence, { root })
  783. ctx.llm.registerAdapter(['mock'], new MockAdapter([textResponse('next')]))
  784. const lateOpen = Promise.withResolvers<SessionHandle>()
  785. const openStarted = Promise.withResolvers<undefined>()
  786. ctx.sessionPersistence.open = (id) => {
  787. expect(id).toBe(sessionId)
  788. openStarted.resolve(undefined)
  789. return lateOpen.promise
  790. }
  791. const published: string[] = []
  792. ctx.on('session/created', () => void published.push('session/created'))
  793. ctx.on('agent/created', () => void published.push('agent/created'))
  794. const resuming = ctx.agents.resume({ resumeSessionId: sessionId, agentOptions: { provider: 'mock', model: 'mock' } })
  795. await openStarted.promise
  796. const rejection = expect(promptly(resuming)).rejects.toThrow(/agent loop is not active/)
  797. await promptly(loopFiber.dispose())
  798. await rejection
  799. expect(published).toEqual([])
  800. expect(ctx.agents.get(sessionId)).toBeUndefined()
  801. expect(ctx.sessions.get(sessionId)).toBeUndefined()
  802. // The abandoned handle is closed once the hung open finally settles.
  803. const abandoned = abandonedHandleStub()
  804. lateOpen.resolve(abandoned.handle)
  805. await expect.poll(() => abandoned.close.mock.calls.length).toBe(1)
  806. expect(published).toEqual([])
  807. await ctx.fiber.dispose()
  808. })
  809. it('resume of a forked session preserves the lineage, seed boundary, and delegation depth in the header', async () => {
  810. // Lifecycle 1: persist a FORKED session (carries parentSession + seedLength
  811. // in its header) through createAgent with a complete-turn seed — the
  812. // factory stores the header and seed through its write handle.
  813. const seed: SessionEvent[] = [
  814. { type: 'turn/start', seq: SessionSeq(0), time: 1, data: { turn: 1 } },
  815. { type: 'turn/end', seq: SessionSeq(1), time: 2, data: { turn: 1, reason: { kind: 'completed' } } },
  816. ]
  817. const { ctx: ctx1, root } = await persistentHarness(new MockAdapter([]))
  818. const forked = await ctx1.agents.create({
  819. sessionId: SessionId('forked-sess'),
  820. seed,
  821. meta: { cwd: '/w', parentSession: SessionId('parent-sess'), isSeeded: true, delegationDepth: 1 },
  822. inheritedEventCount: SessionLogOffset(seed.length),
  823. })
  824. await forked.dispose()
  825. await ctx1.fiber.dispose()
  826. // Lifecycle 2: resume it; the parentSession + seedLength header survives the
  827. // round-trip (exercises resume's parentSession- and seedLength-present
  828. // branches). seedLength must come from the PERSISTED header, not from the
  829. // resume seed length (which is the whole stored log, not the original
  830. // boundary).
  831. const ctx2 = await mountPersistentHarness(root, new MockAdapter([textResponse('b')]))
  832. const a2 = (await ctx2.agents.resume({ resumeSessionId: SessionId('forked-sess') })).agent
  833. expect(a2.session.header.parentSession).toBe('parent-sess')
  834. expect(a2.session.header.cwd).toBe('/w')
  835. expect(a2.session.header.isSeeded).toBe(true)
  836. expect(a2.session.inheritedEventCount).toBe(seed.length)
  837. // The recursion budget survives resume — a dropped depth would let a
  838. // resumed child delegate as if it were top-level.
  839. expect(a2.session.header.delegationDepth).toBe(1)
  840. await ctx2.fiber.dispose()
  841. })
  842. // The crash simulation removes the wedged lifecycle's lock file, which only
  843. // POSIX's orphan-inode forfeit honors; Windows pins the name until the
  844. // process exits, and cross-process crash release is pinned by the jsonl
  845. // two-process e2e.
  846. it.skipIf(process.platform === 'win32')('a pending idle inject() survives persist + resume without a synthetic turn', async () => {
  847. const adapter1 = new MockAdapter([textResponse('answer')])
  848. const { ctx: ctx1, root } = await persistentHarness(adapter1)
  849. const a1 = (await ctx1.agents.create({ sessionId: SessionId('inject-sess'), meta: { cwd: '/w' }, agentOptions: { provider: 'mock', model: 'mock' } })).agent
  850. a1.followup(createUserMessage({ content: [{ type: 'text', text: 'q' }], source: { kind: 'user' } }))
  851. await waitForIdle(ctx1, a1)
  852. a1.inject(createUserMessage({ content: [{ type: 'text', text: 'background job 42 finished' }], source: { kind: 'plugin', plugin: 'tool-bash' } }))
  853. await a1.whenIdle()
  854. await ctx1.sessions.flush(a1.session)
  855. // Simulate a wedged first lifecycle: a graceful dispose would durably
  856. // discard the pending inject, and the still-open kernel write lock would
  857. // otherwise exclude the second lifecycle. Removing the lock file orphans
  858. // the held inode so the resumer locks a fresh one (the documented
  859. // forfeit-by-unlink escape hatch).
  860. await removeSessionLocks(root)
  861. // Lifecycle 2: resume; the injected context is still pending and becomes
  862. // model-visible when the next turn admits it.
  863. const ctx2 = await mountPersistentHarness(root, new MockAdapter([textResponse('next')]))
  864. const stored = await readStoredEvents(ctx2, SessionId('inject-sess'))
  865. expect(stored.some(event => event.type === 'agent/inbox/spliced')).toBe(true)
  866. expect(JSON.stringify(stored)).toContain('background job 42 finished')
  867. const a2 = (await ctx2.agents.resume({ resumeSessionId: SessionId('inject-sess'), agentOptions: { provider: 'mock', model: 'mock' } })).agent
  868. expect(JSON.stringify(a2.inbox.nextStep)).toContain('background job 42 finished')
  869. a2.followup(createUserMessage({ content: [{ type: 'text', text: 'continue' }], source: { kind: 'user' } }))
  870. await waitForIdle(ctx2, a2)
  871. const flat = JSON.stringify(a2.session.deriveMessages())
  872. expect(flat).toContain('background job 42 finished')
  873. await ctx2.fiber.dispose()
  874. await ctx1.fiber.dispose()
  875. })
  876. it('resume reloads a persisted session: history + turn numbering continue, no duplicate seqs', async () => {
  877. // Lifecycle 1: run one full turn, persisting it.
  878. const adapter1 = new MockAdapter([textResponse('first answer')])
  879. const { ctx: ctx1, root } = await persistentHarness(adapter1)
  880. const h1 = await ctx1.agents.create({ sessionId: SessionId('sess-resume'), meta: { cwd: '/w' } })
  881. const a1 = h1.agent
  882. a1.followup(createUserMessage({ content: [{ type: 'text', text: 'first question' }], source: { kind: 'user' } }))
  883. await waitForIdle(ctx1, a1)
  884. const events1 = a1.session.snapshotEvents()
  885. const seqs1 = events1.map(e => e.seq)
  886. expect(seqs1).toEqual([...seqs1].sort((x, y) => x - y)) // contiguous
  887. await h1.dispose()
  888. await ctx1.fiber.dispose()
  889. // Lifecycle 2: a brand-new context over the SAME root; resume the session.
  890. const ctx2 = await mountPersistentHarness(root, new MockAdapter([textResponse('second answer')]))
  891. const a2 = (await ctx2.agents.resume({ resumeSessionId: SessionId('sess-resume') })).agent
  892. // The resumed session carries the prior history…
  893. expect(a2.session.id).toBe('sess-resume')
  894. // …followed by one end-seed event marking the constructor seed.
  895. expect(a2.session.snapshotEvents().length).toBe(events1.length + 1)
  896. expect(a2.session.firstLiveSeq).toBe(events1.length)
  897. expect(a2.session.snapshotEvents().at(-1)?.type).toBe('session/end-seed')
  898. const replay = Session.create(SessionId('replay'), events1)
  899. expect(a2.session.deriveMessages()).toEqual(replay.deriveMessages())
  900. // …and a new turn continues numbering (turn 2) with contiguous seqs.
  901. a2.followup(createUserMessage({ content: [{ type: 'text', text: 'second question' }], source: { kind: 'user' } }))
  902. await waitForIdle(ctx2, a2)
  903. const allSeqs = a2.session.snapshotEvents().map(e => e.seq)
  904. expect(allSeqs).toEqual(allSeqs.map((_, i) => i)) // 0..N contiguous, no duplicates
  905. const turnStarts = a2.session.snapshotEvents().filter(e => e.type === 'turn/start')
  906. expect(turnStarts.map(e => e.type === 'turn/start' && e.data.turn)).toEqual([1, 2])
  907. await ctx2.fiber.dispose()
  908. })
  909. it('resume rejects when session persistence is not configured', async () => {
  910. // A harness WITHOUT the persistence plugin.
  911. const adapter = new MockAdapter([textResponse('x')])
  912. const ctx = new Context()
  913. await ctx.plugin(LlmRuntime)
  914. await ctx.plugin(SessionStore)
  915. await ctx.plugin(SessionProjectionRegistry)
  916. await ctx.plugin(SystemPrompt)
  917. await ctx.plugin(ToolRuntime)
  918. await ctx.plugin(AgentRegistry)
  919. await ctx.plugin(AgentLoop, { agents: [] })
  920. ctx.llm.registerAdapter(['mock'], adapter)
  921. await expect(ctx.agents.resume({ resumeSessionId: SessionId('nope') }))
  922. .rejects.toThrow(/session persistence is not configured/)
  923. await ctx.fiber.dispose()
  924. })
  925. })
  926. describe('creation and resume cancellation edges', () => {
  927. it('rejects create() with a pre-aborted signal, including a non-Error reason', async () => {
  928. const { ctx } = await persistentHarness(new MockAdapter([]))
  929. const errorReason = new AbortController()
  930. errorReason.abort(new Error('caller gave up'))
  931. await expect(promptly(ctx.agents.create({
  932. sessionId: SessionId('pre-aborted-error'),
  933. agentOptions: { provider: 'mock', model: 'mock' },
  934. signal: errorReason.signal,
  935. }))).rejects.toThrow('caller gave up')
  936. // A non-Error reason is wrapped into the creation-aborted error.
  937. const stringReason = new AbortController()
  938. stringReason.abort('operator string reason')
  939. await expect(promptly(ctx.agents.create({
  940. sessionId: SessionId('pre-aborted-string'),
  941. agentOptions: { provider: 'mock', model: 'mock' },
  942. signal: stringReason.signal,
  943. }))).rejects.toThrow(/creation aborted/)
  944. expect(ctx.agents.get(SessionId('pre-aborted-error'))).toBeUndefined()
  945. expect(ctx.agents.get(SessionId('pre-aborted-string'))).toBeUndefined()
  946. await ctx.fiber.dispose()
  947. })
  948. it('a non-Error abort reason arriving during setup is wrapped for the caller', async () => {
  949. const { ctx } = await persistentHarness(new MockAdapter([]))
  950. const controller = new AbortController()
  951. const setupEntered = Promise.withResolvers<undefined>()
  952. const setupGate = Promise.withResolvers<undefined>()
  953. const creating = ctx.agents.create({
  954. sessionId: SessionId('setup-string-abort'),
  955. agentOptions: { provider: 'mock', model: 'mock' },
  956. signal: controller.signal,
  957. async setup() {
  958. setupEntered.resolve(undefined)
  959. await setupGate.promise
  960. },
  961. })
  962. await setupEntered.promise
  963. controller.abort('mid-setup string reason')
  964. setupGate.resolve(undefined)
  965. await expect(promptly(creating)).rejects.toThrow(/creation aborted/)
  966. expect(ctx.agents.get(SessionId('setup-string-abort'))).toBeUndefined()
  967. await ctx.fiber.dispose()
  968. })
  969. it('rejects when setup synchronously aborts its caller signal', async () => {
  970. const { ctx } = await persistentHarness(new MockAdapter([]))
  971. const controller = new AbortController()
  972. const creating = ctx.agents.create({
  973. sessionId: SessionId('setup-synchronous-abort'),
  974. agentOptions: { provider: 'mock', model: 'mock' },
  975. signal: controller.signal,
  976. setup() {
  977. controller.abort(new Error('setup synchronously cancelled'))
  978. },
  979. })
  980. await expect(promptly(creating)).rejects.toThrow('setup synchronously cancelled')
  981. expect(ctx.agents.get(SessionId('setup-synchronous-abort'))).toBeUndefined()
  982. await ctx.fiber.dispose()
  983. })
  984. it('resume with a pre-aborted caller signal rejects out of the load race', async () => {
  985. const sessionId = SessionId('resume-pre-aborted')
  986. const root = await persistSession(sessionId)
  987. const ctx = await mountPersistentHarness(root, new MockAdapter([]))
  988. const controller = new AbortController()
  989. controller.abort(new Error('resume abandoned'))
  990. await expect(promptly(ctx.agents.resume({
  991. resumeSessionId: sessionId,
  992. agentOptions: { provider: 'mock', model: 'mock' },
  993. signal: controller.signal,
  994. }))).rejects.toThrow('resume abandoned')
  995. const stringReason = new AbortController()
  996. stringReason.abort('resume string reason')
  997. await expect(promptly(ctx.agents.resume({
  998. resumeSessionId: sessionId,
  999. agentOptions: { provider: 'mock', model: 'mock' },
  1000. signal: stringReason.signal,
  1001. }))).rejects.toThrow(/creation aborted/)
  1002. expect(ctx.agents.get(sessionId)).toBeUndefined()
  1003. await ctx.fiber.dispose()
  1004. })
  1005. it('an abort landing between crash repair and publication refuses resume, normalized', async () => {
  1006. // Cover the publication-time abort backstop: the signal fires AFTER the
  1007. // raced open settled (during the closer append), so only the final
  1008. // pre-publication check can refuse.
  1009. const run = async (suffix: string, reason: unknown): Promise<unknown> => {
  1010. const sessionId = SessionId(`late-abort-${suffix}`)
  1011. const { ctx: seedCtx, root } = await persistentHarness(new MockAdapter([]))
  1012. await seedStoredSession(seedCtx, sessionId, [
  1013. { type: 'turn/start', seq: SessionSeq(0), time: 1, data: { turn: 1 } },
  1014. ])
  1015. await seedCtx.fiber.dispose()
  1016. const ctx = await mountPersistentHarness(root, new MockAdapter([]))
  1017. const controller = new AbortController()
  1018. const originalOpen = ctx.sessionPersistence.open.bind(ctx.sessionPersistence)
  1019. ctx.sessionPersistence.open = async (id, access, options) => {
  1020. const handle = await originalOpen(id, access, options)
  1021. const realAppend = handle.append.bind(handle)
  1022. Object.defineProperty(handle, 'append', {
  1023. value: (events: readonly SessionEvent[]) => {
  1024. controller.abort(reason)
  1025. return realAppend(events)
  1026. },
  1027. })
  1028. return handle
  1029. }
  1030. const rejection = await ctx.agents.resume({
  1031. resumeSessionId: sessionId,
  1032. agentOptions: { provider: 'mock', model: 'mock' },
  1033. signal: controller.signal,
  1034. }).then(() => undefined, (error: unknown) => error)
  1035. expect(ctx.agents.get(sessionId)).toBeUndefined()
  1036. await ctx.fiber.dispose()
  1037. return rejection
  1038. }
  1039. expect(await run('error', new Error('late abort error'))).toMatchObject({ message: 'late abort error' })
  1040. expect(await run('string', 'late abort string')).toMatchObject({ message: expect.stringMatching(/creation aborted/) as unknown })
  1041. })
  1042. it('an aborted create closes the write handle its abandoned backend create later resolves', async () => {
  1043. const sessionId = SessionId('create-abandoned-handle')
  1044. const { ctx } = await persistentHarness(new MockAdapter([]))
  1045. const lateCreate = Promise.withResolvers<SessionHandle>()
  1046. const createStarted = Promise.withResolvers<undefined>()
  1047. ctx.sessionPersistence.create = () => {
  1048. createStarted.resolve(undefined)
  1049. return lateCreate.promise
  1050. }
  1051. const controller = new AbortController()
  1052. const creating = ctx.agents.create({ sessionId, signal: controller.signal })
  1053. await createStarted.promise
  1054. controller.abort(new Error('caller aborted create'))
  1055. await expect(creating).rejects.toThrow('caller aborted create')
  1056. expect(ctx.sessions.get(sessionId)).toBeUndefined()
  1057. // The abandoned backend create still resolves a real write handle later;
  1058. // the loop closes it so ownership is not leaked, and a rejecting close is
  1059. // swallowed — there is no owner left to observe it.
  1060. const close = vi.fn(async () => { throw new Error('abandoned close failed') })
  1061. lateCreate.resolve({ append: async () => {}, close } as unknown as SessionHandle)
  1062. await expect.poll(() => close.mock.calls.length).toBe(1)
  1063. await ctx.fiber.dispose()
  1064. })
  1065. it('closes the resume write handle when the loop becomes inactive before setup', async () => {
  1066. const sessionId = SessionId('resume-loop-inactive-after-open')
  1067. const root = await persistSession(sessionId)
  1068. const ctx = await mountPersistentHarness(root, new MockAdapter([]))
  1069. const loop = ctx.agentLoop as unknown as {
  1070. ownership: { isActive: () => boolean }
  1071. }
  1072. vi.spyOn(loop.ownership, 'isActive').mockReturnValueOnce(false)
  1073. await expect(ctx.agents.resume({
  1074. resumeSessionId: sessionId,
  1075. agentOptions: { provider: 'mock', model: 'mock' },
  1076. })).rejects.toThrow('agent loop is not active')
  1077. expect(ctx.agents.get(sessionId)).toBeUndefined()
  1078. // The already-open handle was closed on the inactive path: a retry can
  1079. // claim write ownership again.
  1080. const retry = await ctx.agents.resume({
  1081. resumeSessionId: sessionId,
  1082. agentOptions: { provider: 'mock', model: 'mock' },
  1083. })
  1084. await retry.dispose()
  1085. await ctx.fiber.dispose()
  1086. })
  1087. it('factory teardown during a hung resume open rejects promptly and closes the late handle', async () => {
  1088. const sessionId = SessionId('resume-loop-teardown')
  1089. const root = await persistSession(sessionId)
  1090. const ctx = await mountPersistentHarness(root, new MockAdapter([]))
  1091. const gate = Promise.withResolvers<SessionHandle>()
  1092. const openStarted = Promise.withResolvers<undefined>()
  1093. ctx.sessionPersistence.open = () => {
  1094. openStarted.resolve(undefined)
  1095. return gate.promise
  1096. }
  1097. const resuming = ctx.agents.resume({
  1098. resumeSessionId: sessionId,
  1099. agentOptions: { provider: 'mock', model: 'mock' },
  1100. })
  1101. await openStarted.promise
  1102. // Resolve the open only after teardown began: the abandoned handle must
  1103. // still be released even though the wrapper already rejected.
  1104. const rejection = expect(promptly(resuming)).rejects.toThrow()
  1105. const disposal = ctx.fiber.dispose()
  1106. const abandoned = abandonedHandleStub()
  1107. gate.resolve(abandoned.handle)
  1108. await rejection
  1109. await disposal
  1110. await expect.poll(() => abandoned.close.mock.calls.length).toBe(1)
  1111. })
  1112. })
  1113. describe('configured-start failure edges', () => {
  1114. it('a non-Error mid-open abort reason is wrapped for the resume caller', async () => {
  1115. const sessionId = SessionId('resume-string-mid-abort')
  1116. const root = await persistSession(sessionId)
  1117. const ctx = await mountPersistentHarness(root, new MockAdapter([]))
  1118. const gate = Promise.withResolvers<never>()
  1119. gate.promise.catch(() => undefined)
  1120. const openStarted = Promise.withResolvers<undefined>()
  1121. ctx.sessionPersistence.open = () => {
  1122. openStarted.resolve(undefined)
  1123. return gate.promise
  1124. }
  1125. const controller = new AbortController()
  1126. const resuming = ctx.agents.resume({
  1127. resumeSessionId: sessionId,
  1128. agentOptions: { provider: 'mock', model: 'mock' },
  1129. signal: controller.signal,
  1130. })
  1131. await openStarted.promise
  1132. controller.abort('operator string reason')
  1133. await expect(promptly(resuming)).rejects.toThrow(/creation aborted/)
  1134. expect(ctx.agents.get(sessionId)).toBeUndefined()
  1135. await ctx.fiber.dispose()
  1136. })
  1137. it('a failing exact-id restore over an existing artifact stays loud', async () => {
  1138. const sessionId = SessionId('config-existing-corrupt')
  1139. const root = await persistSession(sessionId)
  1140. const configured = new Context()
  1141. await configured.plugin(LlmRuntime)
  1142. await configured.plugin(SessionStore)
  1143. await configured.plugin(SessionProjectionRegistry)
  1144. await configured.plugin(SystemPrompt)
  1145. await configured.plugin(ToolRuntime)
  1146. await configured.plugin(AgentRegistry)
  1147. await configured.plugin(JsonlSessionPersistence, { root })
  1148. configured.llm.registerAdapter(['mock'], new MockAdapter([]))
  1149. // The artifact exists but its open fails with a NON-NotFound error: this is
  1150. // corruption, not first creation — the failure must be reported, and no
  1151. // fresh same-id session may shadow the broken one.
  1152. configured.sessionPersistence.open = () => Promise.reject(new Error('artifact corrupt'))
  1153. const configFailures: unknown[] = []
  1154. configured.on('agent-loop/config-start-failed', ({ error }) => { configFailures.push(error) })
  1155. const warn = vi.spyOn(configured.logger, 'warn').mockImplementation(() => undefined)
  1156. const loop = await configured.plugin(AgentLoop, {
  1157. agents: [{ id: 'main', sessionId, provider: 'mock', model: 'mock' }],
  1158. })
  1159. await expect.poll(() => configFailures.length).toBe(1)
  1160. expect(configFailures[0]).toBeInstanceOf(Error)
  1161. expect((configFailures[0] as Error).message).toBe('artifact corrupt')
  1162. expect(warn).toHaveBeenCalledWith(expect.stringContaining('config-driven restore'))
  1163. expect(configured.agents.get(sessionId)).toBeUndefined()
  1164. warn.mockRestore()
  1165. await loop.dispose()
  1166. await configured.fiber.dispose()
  1167. })
  1168. it('suppresses a configured-resume failure that lands after teardown', async () => {
  1169. const sessionId = SessionId('config-late-resume-failure')
  1170. const root = await persistSession(sessionId)
  1171. const configured = new Context()
  1172. await configured.plugin(LlmRuntime)
  1173. await configured.plugin(SessionStore)
  1174. await configured.plugin(SessionProjectionRegistry)
  1175. await configured.plugin(SystemPrompt)
  1176. await configured.plugin(ToolRuntime)
  1177. await configured.plugin(AgentRegistry)
  1178. await configured.plugin(JsonlSessionPersistence, { root })
  1179. configured.llm.registerAdapter(['mock'], new MockAdapter([]))
  1180. const gate = Promise.withResolvers<never>()
  1181. gate.promise.catch(() => undefined)
  1182. const openStarted = Promise.withResolvers<undefined>()
  1183. configured.sessionPersistence.open = () => {
  1184. openStarted.resolve(undefined)
  1185. return gate.promise
  1186. }
  1187. const failures: unknown[] = []
  1188. configured.on('agent-loop/config-start-failed', ({ error }) => { failures.push(error) })
  1189. const loop = await configured.plugin(AgentLoop, {
  1190. agents: [{ id: 'main', resumeSessionId: sessionId, provider: 'mock', model: 'mock' }],
  1191. })
  1192. await openStarted.promise
  1193. const disposal = loop.dispose()
  1194. gate.reject(new Error('late backend failure'))
  1195. await disposal
  1196. await new Promise(r => setTimeout(r, 20))
  1197. // Ownership deactivated before the failure landed: the report is dropped.
  1198. expect(failures).toEqual([])
  1199. await configured.fiber.dispose()
  1200. })
  1201. })