workflow-workerthread.spec.ts 63 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334
  1. import { describe, expect, it, vi } from 'vitest'
  2. import { fileURLToPath } from 'node:url'
  3. import type { Worker } from 'node:worker_threads'
  4. import { Context } from 'cordis'
  5. import Loader from '@cordisjs/plugin-loader'
  6. import { AgentId } from '@deepseek-ai/dsh-agent'
  7. import type { Agent } from '@deepseek-ai/dsh-agent'
  8. import SubagentService from '@deepseek-ai/dsh-subagent'
  9. import type { SubagentCapabilities, SubagentProvider, SubagentResult, SubagentRun, SubagentStartRequest } from '@deepseek-ai/dsh-subagent'
  10. import type { WorkflowMeta, WorkflowResult, WorkflowResultInfo, WorkflowRunInfo } from '@deepseek-ai/dsh-workflow'
  11. import * as workerEngineModule from '../src/index.ts'
  12. import WorkerWorkflowEngine, { HostToWorkerType, WorkerToHostType, type Config } from '../src/index.ts'
  13. /** A minimal parent stand-in: the engine only threads it through to the provider. */
  14. function fakeParent(): Agent {
  15. return { id: AgentId('workflow-parent'), options: {} } as unknown as Agent
  16. }
  17. // Allow cold worker startup on contended CI runners.
  18. vi.setConfig({ testTimeout: 30_000 })
  19. /**
  20. * Wait up to 10 seconds for CPU-bound worker startup or cross-thread delivery on contended CI.
  21. * Host reactions after an observed event use explicit tight overrides, so this generous startup
  22. * allowance cannot hide multi-second reap regressions.
  23. */
  24. function waitFor(assertion: () => void, timeout = 10_000): Promise<void> {
  25. return vi.waitFor(assertion, { timeout, interval: 50 })
  26. }
  27. /** The vm-context escape hatch, spelled once: real Worker tests use it to make the WORKER misbehave. */
  28. const ESCAPE = "globalThis.constructor.constructor('return process')()"
  29. /** One controllable child run: the test (or auto mode) settles it. */
  30. interface ControlledRun {
  31. request: SubagentStartRequest
  32. /** Fulfill the provider's async start with a ready child. */
  33. publish(): void
  34. /** Reject the provider's async start before ownership transfer. */
  35. rejectStart(error: unknown): void
  36. settle(result: SubagentResult): void
  37. rejectResult(error: unknown): void
  38. cancelled: string | undefined
  39. disposed: boolean
  40. disposeCalls: number
  41. }
  42. /**
  43. * A scripted in-test provider over the REAL SubagentService registry: `auto`
  44. * settles each run via the reply function on a microtask; `manual` piles runs
  45. * up in `runs` for the test to settle. A run aborts (settles `aborted`) when
  46. * the request signal fires, like the real in-process backends.
  47. */
  48. class StubProvider implements SubagentProvider {
  49. readonly capabilities: SubagentCapabilities = { outputSchema: true, depthLimit: true, toolFilter: true, persona: false }
  50. readonly inheritsParentContext = false
  51. readonly runs: ControlledRun[] = []
  52. constructor(
  53. readonly name: string,
  54. private readonly reply?: (request: SubagentStartRequest, index: number) => SubagentResult,
  55. private readonly disposeDelayMs = 0,
  56. private readonly deferStart = false,
  57. private readonly onAbortString?: (reason: string | undefined, index: number) => void,
  58. private readonly onSignalAbort?: (reason: unknown, index: number) => void,
  59. ) {}
  60. async start(request: SubagentStartRequest): Promise<SubagentRun> {
  61. const startGate = Promise.withResolvers<undefined>()
  62. const terminal = Promise.withResolvers<SubagentResult>()
  63. terminal.promise.catch(() => { /* provider owns early settlement until publication */ })
  64. let published = false
  65. const controlled: ControlledRun = {
  66. request,
  67. publish: () => { published = true; startGate.resolve(undefined) },
  68. rejectStart: (error) => { startGate.reject(error) },
  69. settle: (result) => { terminal.resolve(result) },
  70. rejectResult: (error) => { terminal.reject(error) },
  71. cancelled: undefined,
  72. disposed: false,
  73. disposeCalls: 0,
  74. }
  75. this.runs.push(controlled)
  76. const index = this.runs.length - 1
  77. request.signal.addEventListener('abort', () => {
  78. controlled.cancelled = String(request.signal.reason ?? 'cancelled')
  79. this.onAbortString?.(String(request.signal.reason ?? 'cancelled'), index)
  80. this.onSignalAbort?.(request.signal.reason, index)
  81. if (published) terminal.resolve({ output: [], stopReason: 'aborted' })
  82. else startGate.reject(new Error('child start aborted before publication'))
  83. }, { once: true })
  84. if (!this.deferStart) controlled.publish()
  85. if (this.reply) {
  86. const reply = this.reply
  87. queueMicrotask(() => { terminal.resolve(reply(request, index)) })
  88. }
  89. try {
  90. await startGate.promise
  91. } catch (error: unknown) {
  92. controlled.disposeCalls += 1
  93. controlled.disposed = true
  94. throw error
  95. }
  96. if (request.signal.aborted) throw new Error('child start aborted before publication')
  97. return {
  98. id: AgentId(`stub-child-${index}`),
  99. result: terminal.promise,
  100. dispose: () => {
  101. controlled.disposeCalls += 1
  102. if (this.disposeDelayMs === 0) {
  103. controlled.disposed = true
  104. return Promise.resolve()
  105. }
  106. return new Promise<void>((resolve) => {
  107. setTimeout(() => {
  108. controlled.disposed = true
  109. resolve()
  110. }, this.disposeDelayMs)
  111. })
  112. },
  113. }
  114. }
  115. }
  116. /** Text-reply helper for auto providers. */
  117. function text(reply: string): SubagentResult {
  118. return { output: [{ type: 'text', text: reply }], stopReason: 'completed' }
  119. }
  120. interface SetupOptions {
  121. config?: Config
  122. reply?: (request: SubagentStartRequest, index: number) => SubagentResult
  123. manual?: boolean
  124. disposeDelayMs?: number
  125. deferStart?: boolean
  126. onChildAbortString?: (reason: string | undefined, index: number) => void
  127. onChildSignalAbort?: (reason: unknown, index: number) => void
  128. }
  129. async function setup(options?: SetupOptions) {
  130. const ctx = new Context()
  131. await ctx.plugin(SubagentService)
  132. const provider = new StubProvider(
  133. 'stub',
  134. options?.manual ? undefined : options?.reply ?? (() => text('stub reply')),
  135. options?.disposeDelayMs ?? 0,
  136. options?.deferStart ?? false,
  137. options?.onChildAbortString,
  138. options?.onChildSignalAbort,
  139. )
  140. ctx.subagents.registerProvider(provider)
  141. // A fixed concurrency ceiling: the auto-resolved default is machine-derived
  142. // (cores - 2, floored at 1), so tests that expect N children in flight
  143. // would wedge on small CI runners.
  144. const engineFiber = await ctx.plugin(WorkerWorkflowEngine, { provider: 'stub', maxConcurrentAgents: 8, ...options?.config })
  145. return { ctx, provider, parent: fakeParent(), engineFiber }
  146. }
  147. /** The standard test meta plus a body, spread into a start request. */
  148. function scripted(body: string, metaExtra?: Partial<WorkflowMeta>): { script: string; meta: WorkflowMeta } {
  149. return { script: body, meta: { name: 'test-flow', description: 'a test workflow', ...metaExtra } }
  150. }
  151. /** Start + await one run, disposing on the way out. */
  152. async function run(ctx: Context, parent: Agent, source: { script: string; meta: WorkflowMeta }, args?: unknown): Promise<WorkflowResult> {
  153. const handle = ctx.workflows.start({ ...source, parent, ...args !== undefined ? { args } : {} })
  154. try {
  155. return await handle.result
  156. } finally {
  157. await handle.dispose()
  158. }
  159. }
  160. describe('dsh-workflow-workerthread', () => {
  161. describe('script execution over a real worker thread', () => {
  162. it('runs a script end-to-end: agent() text results, phases, log, args, return value, events', async () => {
  163. const { ctx, parent, provider } = await setup({ reply: (_request, index) => text(`answer-${index}`) })
  164. const events: [string, unknown[]][] = []
  165. for (const name of ['workflow/start', 'workflow/phase', 'workflow/log', 'workflow/agent-start', 'workflow/agent-end', 'workflow/end'] as const) {
  166. ctx.on(name, (...payload: unknown[]) => { events.push([name, payload]) })
  167. }
  168. const result = await run(ctx, parent, scripted(`
  169. phase('Scan')
  170. log('starting with ' + args.files.length + ' files')
  171. const answers = await pipeline(args.files, (prev, item) => agent('read ' + item))
  172. phase('Report')
  173. return { answers, count: args.files.length }
  174. `, { phases: [{ title: 'Scan' }, { title: 'Report' }] }), { files: ['a.ts', 'b.ts'] })
  175. expect(result.stopReason).toBe('completed')
  176. expect(result.agentsStarted).toBe(2)
  177. expect(result.value).toEqual({ answers: ['answer-0', 'answer-1'], count: 2 })
  178. expect(provider.runs.every(r => r.disposed)).toBe(true)
  179. const names = events.map(([name]) => name)
  180. expect(names[0]).toBe('workflow/start')
  181. expect(names).toContain('workflow/phase')
  182. expect(names).toContain('workflow/log')
  183. expect(names.at(-1)).toBe('workflow/end')
  184. const info = events[0]![1][0] as WorkflowRunInfo
  185. expect(info.meta.name).toBe('test-flow')
  186. const end = events.at(-1)![1][1] as Record<string, unknown>
  187. expect(end).toEqual({ stopReason: 'completed', agentsStarted: 2 })
  188. expect('value' in end).toBe(false)
  189. })
  190. it('agent({schema, model}) forwards outputSchema and agentOptions to the provider across the thread', async () => {
  191. const { ctx, parent, provider } = await setup({
  192. reply: () => ({ output: [], structured: { files: ['x.ts', 'y.ts'] }, stopReason: 'completed' }),
  193. })
  194. const result = await run(ctx, parent, scripted(`
  195. const found = await agent('list files', { model: 'deepseek-v4-pro', schema: { type: 'object', properties: { files: { type: 'array', items: { type: 'string' } } }, required: ['files'] } })
  196. return { first: found.files[0], count: found.files.length }
  197. `))
  198. expect(result.value).toEqual({ first: 'x.ts', count: 2 })
  199. expect(provider.runs[0]!.request.outputSchema).toEqual({
  200. type: 'object',
  201. properties: { files: { type: 'array', items: { type: 'string' } } },
  202. required: ['files'],
  203. })
  204. expect(provider.runs[0]!.request.agentOptions).toEqual({ model: 'deepseek-v4-pro' })
  205. expect(provider.runs[0]!.request.parent).toBeDefined()
  206. })
  207. it('a fatal hook error inside the worker kills the script and reports the error', async () => {
  208. const { ctx, parent } = await setup()
  209. const result = await run(ctx, parent, scripted("return await parallel([() => agent('x', { isolation: 'worktree' })])"))
  210. expect(result.stopReason).toBe('error')
  211. expect(result.error).toContain('"isolation" is deferred')
  212. })
  213. it('a provider start failure crosses back as a fatal AGENT_START error', async () => {
  214. const { ctx, parent } = await setup({ config: { provider: 'nonexistent' } })
  215. const result = await run(ctx, parent, scripted("return await pipeline([1], () => agent('p'))"))
  216. expect(result.stopReason).toBe('error')
  217. expect(result.error).toContain('agent() could not start a child')
  218. })
  219. it('waits for async provider start before announcing a result that settled early', async () => {
  220. const { ctx, parent, provider } = await setup({ manual: true, deferStart: true })
  221. const order: string[] = []
  222. ctx.on('workflow/agent-start', (_info, agent) => { order.push(`start:${agent.seq}`) })
  223. ctx.on('workflow/agent-end', (_info, agent) => { order.push(`end:${agent.outcome}`) })
  224. ctx.on('workflow/end', () => { order.push('run-end') })
  225. const handle = ctx.workflows.start({ ...scripted("return await agent('p')"), parent })
  226. await waitFor(() => { expect(provider.runs.length).toBe(1) })
  227. const early = text('accepted value')
  228. provider.runs[0]!.settle(early)
  229. // The provider still owns this early result while start is pending.
  230. await new Promise(resolve => setTimeout(resolve, 0))
  231. expect(order).toEqual([])
  232. provider.runs[0]!.publish()
  233. const result = await handle.result
  234. expect(result.value).toBe('accepted value')
  235. expect(order).toEqual(['start:1', 'end:completed', 'run-end'])
  236. await handle.dispose()
  237. expect(provider.runs[0]!.disposeCalls).toBe(1)
  238. })
  239. it('observes an early result rejection but sends ChildStarted before ChildFailed after start fulfills', async () => {
  240. const { ctx, parent, provider } = await setup({ manual: true, deferStart: true })
  241. const lifecycle: string[] = []
  242. ctx.on('workflow/agent-start', () => { lifecycle.push('start') })
  243. ctx.on('workflow/agent-end', (_info, agent) => { lifecycle.push(`end:${agent.outcome}`) })
  244. const handle = ctx.workflows.start({
  245. ...scripted("try { await agent('p'); return 'unreachable' } catch (e) { return { code: e.code, message: e.message } }"),
  246. parent,
  247. })
  248. const worker = (handle as unknown as { worker: { postMessage(message: unknown): void } }).worker
  249. const post = vi.spyOn(worker, 'postMessage')
  250. const childMessageTypes = (): HostToWorkerType[] => post.mock.calls
  251. .map(([message]) => (message as { type: HostToWorkerType }).type)
  252. .filter(type => type === HostToWorkerType.ChildStarted || type === HostToWorkerType.ChildFailed)
  253. await waitFor(() => { expect(provider.runs.length).toBe(1) })
  254. provider.runs[0]!.rejectResult(new Error('backend failed before publication'))
  255. await new Promise(resolve => setTimeout(resolve, 0))
  256. expect(childMessageTypes()).toEqual([])
  257. expect(lifecycle).toEqual([])
  258. provider.runs[0]!.publish()
  259. const result = await handle.result
  260. expect(result.value).toMatchObject({ code: 'AGENT_RESULT' })
  261. expect((result.value as { message: string }).message).toContain('backend failed before publication')
  262. expect(childMessageTypes()).toEqual([HostToWorkerType.ChildStarted, HostToWorkerType.ChildFailed])
  263. expect(lifecycle).toEqual(['start', 'end:failed'])
  264. post.mockRestore()
  265. await handle.dispose()
  266. })
  267. it('classifies provider start rejection as AGENT_START, drops an early result, and emits no false lifecycle pair', async () => {
  268. const { ctx, parent, provider } = await setup({ manual: true, deferStart: true })
  269. const lifecycle: string[] = []
  270. ctx.on('workflow/agent-start', () => { lifecycle.push('start') })
  271. ctx.on('workflow/agent-end', () => { lifecycle.push('end') })
  272. const handle = ctx.workflows.start({
  273. ...scripted("try { await agent('p'); return 'unreachable' } catch (e) { return { code: e.code, message: e.message } }"),
  274. parent,
  275. })
  276. await waitFor(() => { expect(provider.runs.length).toBe(1) })
  277. // ACP-style failure can settle result(error) before its session/publication
  278. // boundary rejects. Start rejection must dominate that buffered child outcome.
  279. provider.runs[0]!.settle({ output: [], stopReason: 'error' })
  280. await new Promise(resolve => setTimeout(resolve, 0))
  281. provider.runs[0]!.rejectStart(new Error('publication rolled back'))
  282. const result = await handle.result
  283. expect(result.value).toMatchObject({ code: 'AGENT_START' })
  284. expect((result.value as { message: string }).message).toContain('publication rolled back')
  285. expect(lifecycle).toEqual([])
  286. await waitFor(() => {
  287. expect(provider.runs[0]!.disposed).toBe(true)
  288. expect(provider.runs[0]!.disposeCalls).toBe(1)
  289. })
  290. await handle.dispose()
  291. expect(provider.runs[0]!.disposeCalls).toBe(1)
  292. })
  293. it('aborts a pending provider start once without publishing workflow lifecycle', async () => {
  294. const { ctx, parent, provider } = await setup({ manual: true, deferStart: true, config: { disposeGraceMs: 500 } })
  295. const lifecycle: string[] = []
  296. ctx.on('workflow/agent-start', () => { lifecycle.push('start') })
  297. ctx.on('workflow/agent-end', () => { lifecycle.push('end') })
  298. const handle = ctx.workflows.start({ ...scripted("return await agent('pending')"), parent })
  299. await waitFor(() => { expect(provider.runs.length).toBe(1) })
  300. const disposal = handle.dispose()
  301. await waitFor(() => {
  302. expect(provider.runs[0]!.cancelled).toBe('workflow disposed')
  303. expect(provider.runs[0]!.disposed).toBe(true)
  304. })
  305. // Ensure the host-driven disposal removed the registry entry before the
  306. // late start rejection; its callback must not invoke dispose again.
  307. await new Promise(resolve => setTimeout(resolve, 0))
  308. provider.runs[0]!.rejectStart(new Error('cancelled before publication'))
  309. const result = await handle.result
  310. await disposal
  311. expect(result.stopReason).toBe('cancelled')
  312. expect(lifecycle).toEqual([])
  313. expect(provider.runs[0]!.disposeCalls).toBe(1)
  314. })
  315. it('a child result REJECTION crosses back as a fatal AGENT_RESULT error (a broken provider is not a failed child)', async () => {
  316. const ctx = new Context()
  317. await ctx.plugin(SubagentService)
  318. const provider: SubagentProvider = {
  319. name: 'rejecting',
  320. capabilities: { outputSchema: true, depthLimit: true, toolFilter: true, persona: false },
  321. inheritsParentContext: false,
  322. start: async () => ({
  323. id: AgentId('reject-child'),
  324. result: Promise.reject(new Error('backend exploded')),
  325. dispose: () => Promise.resolve(),
  326. }),
  327. }
  328. ctx.subagents.registerProvider(provider)
  329. await ctx.plugin(WorkerWorkflowEngine, { provider: 'rejecting', maxConcurrentAgents: 2 })
  330. const result = await run(ctx, fakeParent(), scripted(`
  331. try { await agent('p'); return 'unreachable' } catch (e) { return { name: e.name, code: e.code, fatal: e.fatal, message: e.message } }
  332. `))
  333. expect(result.value).toMatchObject({ name: 'WorkflowError', code: 'AGENT_RESULT', fatal: true })
  334. expect((result.value as { message: string }).message).toContain('backend exploded')
  335. })
  336. it('maps a non-JSON ready-child result to fatal AGENT_RESULT instead of wedging the bridge', async () => {
  337. const { ctx, parent } = await setup({
  338. reply: () => ({ output: [], structured: () => { /* deliberately outside lossless JSON */ }, stopReason: 'completed' }),
  339. })
  340. const result = await run(ctx, parent, scripted(`
  341. try { await agent('p'); return 'unreachable' } catch (e) { return { code: e.code, message: e.message } }
  342. `))
  343. expect(result.value).toMatchObject({ code: 'AGENT_RESULT' })
  344. expect((result.value as { message: string }).message).toContain('workflow child result could not cross the worker boundary')
  345. })
  346. it('contains a non-JSON result even if the injected subagent service violates its normalization contract', async () => {
  347. // The real worker boundary must reject a non-JSON same-process result.
  348. const { ctx, parent } = await setup()
  349. const invalid = {
  350. output: [],
  351. structured: () => { /* deliberately outside lossless JSON */ },
  352. stopReason: 'completed',
  353. } as unknown as SubagentResult
  354. const start = vi.spyOn(ctx.subagents, 'start').mockResolvedValue({
  355. id: AgentId('raw-invalid-child'),
  356. result: Promise.resolve(invalid),
  357. dispose: () => Promise.resolve(),
  358. })
  359. const result = await run(ctx, parent, scripted(`
  360. try { await agent('p'); return 'unreachable' } catch (e) { return { code: e.code, message: e.message } }
  361. `))
  362. expect(start).toHaveBeenCalledOnce()
  363. expect(result.value).toMatchObject({ code: 'AGENT_RESULT' })
  364. expect((result.value as { message: string }).message)
  365. .toContain('workflow child result could not cross the worker boundary')
  366. })
  367. it('a child whose dispose() throws synchronously cannot wedge the script (the host acks anyway)', async () => {
  368. const ctx = new Context()
  369. await ctx.plugin(SubagentService)
  370. const provider: SubagentProvider = {
  371. name: 'bad-dispose',
  372. capabilities: { outputSchema: true, depthLimit: true, toolFilter: true, persona: false },
  373. inheritsParentContext: false,
  374. start: async () => ({
  375. id: AgentId('bad-dispose-child'),
  376. result: Promise.resolve({ output: [{ type: 'text', text: 'fine' }], stopReason: 'completed' }),
  377. cancel: () => { /* settled already */ },
  378. dispose: () => { throw new Error('dispose exploded') },
  379. }),
  380. }
  381. ctx.subagents.registerProvider(provider)
  382. await ctx.plugin(WorkerWorkflowEngine, { provider: 'bad-dispose', maxConcurrentAgents: 2 })
  383. const result = await run(ctx, fakeParent(), scripted("return await agent('p')"))
  384. expect(result.stopReason).toBe('completed')
  385. expect(result.value).toBe('fine')
  386. })
  387. it('a child dispose() rejecting an UNRENDERABLE value still acks — the containment warn is total', async () => {
  388. const ctx = new Context()
  389. await ctx.plugin(SubagentService)
  390. const provider: SubagentProvider = {
  391. name: 'coercion-trap-dispose',
  392. capabilities: { outputSchema: true, depthLimit: true, toolFilter: true, persona: false },
  393. inheritsParentContext: false,
  394. start: async () => ({
  395. id: AgentId('trap-child'),
  396. result: Promise.resolve({ output: [{ type: 'text', text: 'fine' }], stopReason: 'completed' }),
  397. cancel: () => { /* settled already */ },
  398. // The rejection VALUE's own coercion throws: a warn built with bare
  399. // String(error) would itself throw, skipping the ChildDisposed ack
  400. // and wedging the script's finally until the grace/terminate path.
  401. // eslint-disable-next-line @typescript-eslint/prefer-promise-reject-errors -- the non-Error rejection IS the scenario under test
  402. dispose: () => Promise.reject({ toString: () => { throw new Error('coercion trap') } }),
  403. }),
  404. }
  405. ctx.subagents.registerProvider(provider)
  406. await ctx.plugin(WorkerWorkflowEngine, { provider: 'coercion-trap-dispose', maxConcurrentAgents: 2 })
  407. const result = await run(ctx, fakeParent(), scripted("return await agent('p')"))
  408. expect(result.stopReason).toBe('completed')
  409. expect(result.value).toBe('fine')
  410. })
  411. it('the worker spawns with an EMPTY environment: an escaped script finds no ambient credentials', async () => {
  412. const { ctx, parent } = await setup()
  413. // A canary in the HARNESS process's env: with an inherited environment
  414. // the escape below would read it back (exactly how DEEPSEEK_API_KEY
  415. // would leak); env: {} in the spawn options is what keeps it out.
  416. process.env.WORKFLOW_ENV_CANARY = 'leak me'
  417. try {
  418. const result = await run(ctx, parent, scripted(`
  419. const proc = ${ESCAPE}
  420. return { canary: proc.env.WORKFLOW_ENV_CANARY ?? null, keys: Object.keys(proc.env).length }
  421. `))
  422. expect(result.stopReason).toBe('completed')
  423. expect(result.value).toEqual({ canary: null, keys: 0 })
  424. } finally {
  425. delete process.env.WORKFLOW_ENV_CANARY
  426. }
  427. })
  428. it('the unbuilt worker forwards exactly TSX_TSCONFIG_PATH through the scrub: the paths-map pin survives, secrets do not', async () => {
  429. const { ctx, parent } = await setup()
  430. // The ACP snapshot harness runs the parent with its cwd OUTSIDE the
  431. // repo and pins the repo tsconfig through this variable; the worker
  432. // must inherit the pin (or its dsh-* imports silently resolve to
  433. // unbuilt lib/ bundles) while every other variable stays scrubbed.
  434. const tsconfig = fileURLToPath(new URL('../../../../tsconfig.json', import.meta.url))
  435. process.env.TSX_TSCONFIG_PATH = tsconfig
  436. process.env.WORKFLOW_ENV_CANARY = 'leak me'
  437. try {
  438. const result = await run(ctx, parent, scripted(`
  439. const proc = ${ESCAPE}
  440. return { keys: Object.keys(proc.env), tsconfig: proc.env.TSX_TSCONFIG_PATH }
  441. `))
  442. expect(result.stopReason).toBe('completed')
  443. expect(result.value).toEqual({ keys: ['TSX_TSCONFIG_PATH'], tsconfig })
  444. } finally {
  445. delete process.env.TSX_TSCONFIG_PATH
  446. delete process.env.WORKFLOW_ENV_CANARY
  447. }
  448. })
  449. })
  450. describe('lifecycle: parse errors, cancellation, termination, disposal', () => {
  451. it('start() throws synchronously for invalid meta data or an unparseable body (host-side pre-checks)', async () => {
  452. const { ctx, parent } = await setup()
  453. // Meta is DATA — shape violations reject loud, every one named.
  454. expect(() => ctx.workflows.start({ script: 'return 1', meta: { name: '', description: 'd' }, parent })).toThrow(/meta\.name must be a non-empty string/)
  455. expect(() => ctx.workflows.start({ script: 'return 1', meta: { name: 'x', description: 'd', extra: 1 } as unknown as WorkflowMeta, parent })).toThrow(/META_INVALID|not a recognized field/)
  456. expect(() => ctx.workflows.start({ ...scripted('return ((('), parent })).toThrow(/does not parse/)
  457. // The likeliest authoring slip — a Claude Code-style meta header in the
  458. // body — gets a pointed message, not a bare SyntaxError.
  459. expect(() => ctx.workflows.start({ ...scripted("export const meta = { name: 'x', description: 'd' }\nreturn 1"), parent })).toThrow(/meta rides the `meta` request field/)
  460. })
  461. it('cancel() aborts in-flight children (signal AND cancel RPC) and settles the run cancelled', async () => {
  462. const { ctx, parent, provider } = await setup({ manual: true })
  463. const ends: unknown[] = []
  464. ctx.on('workflow/agent-end', (_info, agent) => { ends.push(agent) })
  465. const runEnds: WorkflowResultInfo[] = []
  466. ctx.on('workflow/end', (_info, result) => { runEnds.push(result) })
  467. const handle = ctx.workflows.start({ ...scripted("return await agent('long job')"), parent })
  468. await waitFor(() => { expect(provider.runs.length).toBe(1) })
  469. handle.cancel('user stopped it')
  470. const result = await handle.result
  471. expect(result.stopReason).toBe('cancelled')
  472. expect(result.error).toContain('user stopped it')
  473. await handle.dispose()
  474. expect(provider.runs[0]!.disposed).toBe(true)
  475. expect(ends).toEqual([expect.objectContaining({ seq: 1, outcome: 'cancelled' })])
  476. // workflow/end is an observer's only death signal: it fires for a
  477. // cancelled run too, mirroring the settled outcome data.
  478. expect(runEnds).toEqual([{ stopReason: 'cancelled', error: result.error, agentsStarted: result.agentsStarted }])
  479. })
  480. it('an already-aborted request signal cancels before the body ever runs (the go handshake holds it)', async () => {
  481. const { ctx, parent, provider } = await setup()
  482. const controller = new AbortController()
  483. controller.abort()
  484. const logs: string[] = []
  485. ctx.on('workflow/log', (_info, message) => { logs.push(message) })
  486. const handle = ctx.workflows.start({ ...scripted("log('ran')\nreturn 123"), parent, signal: controller.signal })
  487. const result = await handle.result
  488. expect(result.stopReason).toBe('cancelled')
  489. expect(result.value).toBeNull()
  490. expect(logs).toEqual([])
  491. expect(provider.runs.length).toBe(0)
  492. await handle.dispose()
  493. })
  494. it('cancel() right after start() cancels before the body runs; the signal aborting mid-run cancels like cancel()', async () => {
  495. const { ctx, parent, provider } = await setup({ manual: true })
  496. const first = ctx.workflows.start({ ...scripted("return await agent('never')"), parent })
  497. // No-reason cancel: the canonical default reason must ride the result.
  498. first.cancel()
  499. const firstResult = await first.result
  500. expect(firstResult.stopReason).toBe('cancelled')
  501. expect(firstResult.error).toContain('workflow cancelled')
  502. expect(provider.runs.length).toBe(0)
  503. await first.dispose()
  504. const controller = new AbortController()
  505. const second = ctx.workflows.start({ ...scripted("return await agent('job')"), parent, signal: controller.signal })
  506. await waitFor(() => { expect(provider.runs.length).toBe(1) })
  507. controller.abort()
  508. expect((await second.result).stopReason).toBe('cancelled')
  509. await second.dispose()
  510. })
  511. it('removes the exact external abort callback on first settlement or teardown', async () => {
  512. const { ctx, parent } = await setup()
  513. const settledController = new AbortController()
  514. const settledAdd = vi.spyOn(settledController.signal, 'addEventListener')
  515. const settledRemove = vi.spyOn(settledController.signal, 'removeEventListener')
  516. const completed = ctx.workflows.start({ ...scripted('return 123'), parent, signal: settledController.signal })
  517. const settledAbort = settledAdd.mock.calls.find(([type]) => type === 'abort')?.[1]
  518. expect(typeof settledAbort).toBe('function')
  519. await expect(completed.result).resolves.toMatchObject({ value: 123, stopReason: 'completed' })
  520. expect(settledRemove).toHaveBeenCalledWith('abort', settledAbort)
  521. const cancelAfterSettle = vi.spyOn(completed, 'cancel')
  522. settledController.abort()
  523. expect(cancelAfterSettle).not.toHaveBeenCalled()
  524. cancelAfterSettle.mockRestore()
  525. await completed.dispose()
  526. const manual = await setup({ manual: true })
  527. const teardownController = new AbortController()
  528. const teardownAdd = vi.spyOn(teardownController.signal, 'addEventListener')
  529. const teardownRemove = vi.spyOn(teardownController.signal, 'removeEventListener')
  530. const tornDown = manual.ctx.workflows.start({
  531. ...scripted("return await agent('job')"),
  532. parent: manual.parent,
  533. signal: teardownController.signal,
  534. })
  535. await waitFor(() => { expect(manual.provider.runs).toHaveLength(1) })
  536. const teardownAbort = teardownAdd.mock.calls.find(([type]) => type === 'abort')?.[1]
  537. expect(typeof teardownAbort).toBe('function')
  538. const disposing = tornDown.dispose()
  539. expect(teardownRemove).toHaveBeenCalledWith('abort', teardownAbort)
  540. await disposing
  541. })
  542. it('a child-start racing the host cancel is refused: no child starts after cancellation', async () => {
  543. const { ctx, parent, provider } = await setup({ manual: true })
  544. // Cancel from INSIDE the log listener: the worker has already posted
  545. // its child-start (queued right behind the log message), so the host
  546. // processes it with cancelReason set — the refusal arm no real-world
  547. // timing can hit reliably. (The closure runs only after `handle` below
  548. // is initialized — the listener fires on the worker's first message.)
  549. ctx.on('workflow/log', () => { handle.cancel('cancelled from the log listener') })
  550. const handle = ctx.workflows.start({ ...scripted("log('mark')\nreturn await agent('late')"), parent })
  551. const result = await handle.result
  552. expect(result.stopReason).toBe('cancelled')
  553. expect(provider.runs.length).toBe(0)
  554. await handle.dispose()
  555. })
  556. it('post-cancel narration is suppressed host-side, and completion racing a cancel reports cancelled', async () => {
  557. const { ctx, parent } = await setup()
  558. const narration: string[] = []
  559. ctx.on('workflow/log', (_info, message) => { narration.push(message) })
  560. ctx.on('workflow/phase', (_info, title) => { narration.push(`phase:${title}`) })
  561. const handle = ctx.workflows.start({
  562. // The sync spin keeps the worker's loop busy so the cancel message
  563. // cannot be processed before the script settles `completed` — the
  564. // worker posts a completed result that must LOSE to the in-flight
  565. // host cancellation. The trailing narration exercises host-side
  566. // suppression: posted pre-cancel-processing worker-side, arriving
  567. // post-cancel host-side.
  568. ...scripted(`
  569. log('started')
  570. const end = Date.now() + 1000
  571. while (Date.now() < end) {}
  572. phase('late phase')
  573. log('late log')
  574. return 'done'
  575. `),
  576. parent,
  577. })
  578. await waitFor(() => { expect(narration).toContain('started') })
  579. handle.cancel('raced the completion')
  580. const result = await handle.result
  581. expect(result.stopReason).toBe('cancelled')
  582. expect(result.error).toContain('raced the completion')
  583. expect(narration).toEqual(['started'])
  584. await handle.dispose()
  585. }, 15_000)
  586. it('cancel() force-settles a script parked on a promise no hook owns, and TERMINATES its worker', async () => {
  587. const { ctx, parent } = await setup({ config: { provider: 'stub', disposeGraceMs: 50 } })
  588. const runEnds: WorkflowResultInfo[] = []
  589. ctx.on('workflow/end', (_info, result) => { runEnds.push(result) })
  590. const handle = ctx.workflows.start({
  591. ...scripted("await new Promise(() => {})\nreturn 'unreachable'"),
  592. parent,
  593. })
  594. handle.cancel('user aborted')
  595. const result = await handle.result
  596. expect(result.stopReason).toBe('cancelled')
  597. expect(result.error).toContain('user aborted')
  598. // The grace force-settle fires workflow/end exactly like an ordinary
  599. // settlement — a terminated script's death still reaches observers.
  600. expect(runEnds).toEqual([{ stopReason: 'cancelled', error: result.error, agentsStarted: 0 }])
  601. await handle.dispose()
  602. })
  603. it('dispose() on a stuck script returns within the grace instead of hanging (result settles cancelled)', async () => {
  604. const { ctx, parent } = await setup({ config: { provider: 'stub', disposeGraceMs: 50 } })
  605. const handle = ctx.workflows.start({
  606. ...scripted("await new Promise(() => {})\nreturn 'unreachable'"),
  607. parent,
  608. })
  609. const before = Date.now()
  610. await handle.dispose()
  611. expect(Date.now() - before).toBeLessThan(2000)
  612. const result = await handle.result
  613. expect(result.stopReason).toBe('cancelled')
  614. })
  615. it('dispose() is idempotent and settles cleanly after a completed run', async () => {
  616. const { ctx, parent } = await setup()
  617. const handle = ctx.workflows.start({ ...scripted('return 1'), parent })
  618. await handle.result
  619. await handle.dispose()
  620. await handle.dispose()
  621. })
  622. it('a settled run arms NO grace timer: disposing a completed run must not pin it for disposeGraceMs', async () => {
  623. // A distinctive grace so the spy can tell the cancel-path grace timer
  624. // apart from every other timeout in flight.
  625. const GRACE = 44_444
  626. const { ctx, parent } = await setup({ config: { provider: 'stub', disposeGraceMs: GRACE } })
  627. const handle = ctx.workflows.start({ ...scripted('return 1'), parent })
  628. await handle.result
  629. const spy = vi.spyOn(globalThis, 'setTimeout')
  630. try {
  631. await handle.dispose()
  632. // dispose()'s own bounded-wait sleep is the ONLY grace-sized timer
  633. // allowed here; before the settled guard, cancel() armed a second one
  634. // that nothing would ever clear (the run was already settled), keeping
  635. // the WorkerRun/Worker closure alive until the grace expired.
  636. const graceTimers = spy.mock.calls.filter(call => call[1] === GRACE)
  637. expect(graceTimers.length).toBe(1)
  638. } finally {
  639. spy.mockRestore()
  640. }
  641. })
  642. it('strays: children fired without await are aborted once the script settles, and dispose() waits for their disposal', async () => {
  643. const { ctx, parent, provider } = await setup({ manual: true, disposeDelayMs: 40 })
  644. const handle = ctx.workflows.start({
  645. ...scripted(`
  646. agent('stray')
  647. return 'done without awaiting'
  648. `),
  649. parent,
  650. })
  651. const result = await handle.result
  652. expect(result.stopReason).toBe('completed')
  653. await waitFor(() => { expect(provider.runs.length).toBe(1) })
  654. await handle.dispose()
  655. // Not a waitFor: by the time dispose() returns, the slow child disposal
  656. // must already be complete (host-side registry quiescence).
  657. expect(provider.runs[0]!.disposed).toBe(true)
  658. })
  659. it('result settlement reaps a registered stray even when the worker cannot relay disposal', async () => {
  660. const { ctx, parent, provider } = await setup({
  661. manual: true,
  662. config: { provider: 'stub', disposeGraceMs: 30_000 },
  663. })
  664. const handle = ctx.workflows.start({
  665. ...scripted("agent('stray')\nawait new Promise(() => {})"),
  666. parent,
  667. })
  668. await waitFor(() => { expect(provider.runs).toHaveLength(1) })
  669. // Claim the host result while the real worker remains wedged, so it can
  670. // send neither ChildDispose nor an exit. This leaves the accepted child
  671. // in the host registry when public disposal begins.
  672. const worker = (handle as unknown as { worker: Worker }).worker
  673. worker.emit('message', {
  674. type: WorkerToHostType.Result,
  675. result: { value: 'synthetic completion', stopReason: 'completed', agentsStarted: 1 },
  676. })
  677. await expect(handle.result).resolves.toMatchObject({ stopReason: 'completed' })
  678. await waitFor(() => { expect(provider.runs[0]!.disposed).toBe(true) }, 1000)
  679. const disposal = handle.dispose()
  680. await disposal
  681. expect(provider.runs[0]!.disposeCalls).toBe(1)
  682. await ctx.fiber.dispose()
  683. })
  684. it('the settle-reap fires the request signal too: a provider honoring ONLY the signal winds its stray down promptly', async () => {
  685. const ctx = new Context()
  686. await ctx.plugin(SubagentService)
  687. const aborted: string[] = []
  688. const provider: SubagentProvider = {
  689. name: 'signal-only',
  690. capabilities: { outputSchema: true, depthLimit: true, toolFilter: true, persona: false },
  691. inheritsParentContext: false,
  692. start: async (request) => {
  693. let settle!: (result: SubagentResult) => void
  694. const result = new Promise<SubagentResult>((resolve) => { settle = resolve })
  695. request.signal.addEventListener('abort', () => {
  696. aborted.push(String(request.signal.reason))
  697. settle({ output: [], stopReason: 'aborted' })
  698. }, { once: true })
  699. return {
  700. id: AgentId('signal-only-child'),
  701. result,
  702. dispose: () => Promise.resolve(),
  703. }
  704. },
  705. }
  706. ctx.subagents.registerProvider(provider)
  707. await ctx.plugin(WorkerWorkflowEngine, { provider: 'signal-only', maxConcurrentAgents: 2 })
  708. const handle = ctx.workflows.start({
  709. ...scripted(`
  710. agent('stray, never awaited')
  711. return 'done'
  712. `),
  713. parent: fakeParent(),
  714. })
  715. const result = await handle.result
  716. expect(result.stopReason).toBe('completed')
  717. // BEFORE dispose(): the settlement itself must have aborted the signal —
  718. // without it this child would stay live until dispose's terminate. This
  719. // is a HOST-PROMPTNESS claim, not a cold-start race — a tight explicit
  720. // bound (unlike the file default) so a multi-second reap regression
  721. // cannot pass by outlasting the wait.
  722. await waitFor(() => { expect(aborted).toEqual(['workflow settled']) }, 1000)
  723. await handle.dispose()
  724. })
  725. it('the settle-reap aborts a pending provider start before workflow/end', async () => {
  726. const { ctx, parent, provider } = await setup({ manual: true, deferStart: true })
  727. const childLifecycle: string[] = []
  728. let cancellationAtWorkflowEnd: string | undefined
  729. ctx.on('workflow/agent-start', () => { childLifecycle.push('start') })
  730. ctx.on('workflow/agent-end', () => { childLifecycle.push('end') })
  731. ctx.on('workflow/end', () => {
  732. cancellationAtWorkflowEnd = provider.runs[0]?.cancelled
  733. })
  734. const handle = ctx.workflows.start({
  735. ...scripted(`
  736. agent('start-pending stray')
  737. return 'done'
  738. `),
  739. parent,
  740. })
  741. const result = await handle.result
  742. expect(result.stopReason).toBe('completed')
  743. expect(provider.runs).toHaveLength(1)
  744. expect(provider.runs[0]!.request.signal?.aborted).toBe(true)
  745. expect(provider.runs[0]!.request.signal?.reason).toBe('workflow settled')
  746. expect(provider.runs[0]!.cancelled).toBe('workflow settled')
  747. expect(cancellationAtWorkflowEnd).toBe('workflow settled')
  748. expect(childLifecycle).toEqual([])
  749. await handle.dispose()
  750. expect(provider.runs[0]!.disposeCalls).toBe(1)
  751. })
  752. it('a duplicate Result after the terminal claim cannot repeat cleanup or rewrite the outcome', async () => {
  753. let signalAborts = 0
  754. const { ctx, parent, provider } = await setup({
  755. manual: true,
  756. onChildAbortString: (_reason, index) => { if (index === 0) signalAborts += 1 },
  757. })
  758. const handle = ctx.workflows.start({
  759. ...scripted("agent('stray')\nawait new Promise(() => {})"),
  760. parent,
  761. })
  762. await waitFor(() => { expect(provider.runs).toHaveLength(1) })
  763. const worker = (handle as unknown as { worker: Worker }).worker
  764. worker.emit('message', {
  765. type: WorkerToHostType.Result,
  766. result: { value: 'first', stopReason: 'completed', agentsStarted: 1 },
  767. })
  768. worker.emit('message', {
  769. type: WorkerToHostType.Result,
  770. result: { value: 'late', stopReason: 'completed', agentsStarted: 1 },
  771. })
  772. await expect(handle.result).resolves.toMatchObject({ value: 'first', stopReason: 'completed' })
  773. expect(signalAborts).toBe(1)
  774. await handle.dispose()
  775. expect(signalAborts).toBe(1)
  776. await ctx.fiber.dispose()
  777. })
  778. it('a grace-terminated worker reaps its child on exit without waiting for consumer dispose()', async () => {
  779. const { ctx, parent, provider } = await setup({
  780. manual: true,
  781. config: { provider: 'stub', maxConcurrentAgents: 2, disposeGraceMs: 100 },
  782. })
  783. const handle = ctx.workflows.start({
  784. // Let child-start cross, then make the worker unable to process its
  785. // Cancel message. Grace settles the result and terminates the thread;
  786. // that exit must independently own the host registry's disposal pass.
  787. ...scripted(`
  788. agent('survives until exit reap')
  789. for (let i = 0; i < 20; i++) await null
  790. const end = Date.now() + 1500
  791. while (Date.now() < end) {}
  792. return 'unreachable'
  793. `),
  794. parent,
  795. })
  796. await waitFor(() => { expect(provider.runs).toHaveLength(1) })
  797. handle.cancel('force termination')
  798. const result = await handle.result
  799. expect(result.stopReason).toBe('cancelled')
  800. // Deliberately assert before handle.dispose(): host-owned worker exit,
  801. // not consumer courtesy, is responsible for this resource guarantee.
  802. await waitFor(() => { expect(provider.runs[0]!.disposed).toBe(true) }, 1000)
  803. expect(provider.runs[0]!.disposeCalls).toBe(1)
  804. await handle.dispose()
  805. await ctx.fiber.dispose()
  806. }, 15_000)
  807. it('dispose() on a wedged worker host-drives child disposal inside the grace: it returns with the children DISPOSED, not with their teardown still in flight', async () => {
  808. const { ctx, parent, provider } = await setup({
  809. manual: true,
  810. disposeDelayMs: 40,
  811. config: { provider: 'stub', maxConcurrentAgents: 8, disposeGraceMs: 400 },
  812. })
  813. const handle = ctx.workflows.start({
  814. // Same shape as the wedged-cancel test above: the child's start RPC
  815. // reaches the host, then the script seizes its worker's loop, so the
  816. // worker can relay NO dispose RPC — the host's own dispose() drive is
  817. // the only thing that can start (and finish) this child's disposal
  818. // before the grace runs out.
  819. ...scripted(`
  820. agent('wedged child')
  821. for (let i = 0; i < 20; i++) await null
  822. const end = Date.now() + 1500
  823. while (Date.now() < end) {}
  824. return 'raced'
  825. `),
  826. parent,
  827. })
  828. await waitFor(() => { expect(provider.runs.length).toBe(1) })
  829. const before = Date.now()
  830. await handle.dispose()
  831. // Bounded by the grace (plus the terminate), never by the 1.5s spin.
  832. expect(Date.now() - before).toBeLessThan(1200)
  833. // Not a waitFor: dispose() resolving IS the quiescence claim — the slow
  834. // child disposal must be complete, not merely started (before the
  835. // host-driven drive, disposal only STARTED at the post-terminate reap,
  836. // so dispose() returned with it still in flight).
  837. expect(provider.runs[0]!.disposed).toBe(true)
  838. const result = await handle.result
  839. expect(result.stopReason).toBe('cancelled')
  840. }, 15_000)
  841. it('a live child disposed by the dispose() drive is disposed ONCE, and the worker\'s late dispose RPC still gets its ack (the script settles, not the grace)', async () => {
  842. const { ctx, parent, provider } = await setup({ manual: true })
  843. const handle = ctx.workflows.start({
  844. ...scripted(`
  845. await agent('long child')
  846. return 'unreachable'
  847. `),
  848. parent,
  849. })
  850. await waitFor(() => { expect(provider.runs.length).toBe(1) })
  851. const handleDispose = handle.dispose()
  852. const result = await handle.result
  853. // The script itself settled (the wrapper's own dispose RPC found the
  854. // child already reaped host-side and was acked) — a missing ack would
  855. // wedge the wrapper's finally until the 5s default grace force-settle.
  856. expect(result.stopReason).toBe('cancelled')
  857. expect(result.error).toContain('workflow disposed')
  858. await handleDispose
  859. expect(provider.runs[0]!.disposed).toBe(true)
  860. // The memo: the host drive and the worker's RPC share one disposal.
  861. expect(provider.runs[0]!.disposeCalls).toBe(1)
  862. })
  863. it('the grace force-settle pairs every stranded start: a host-synthesized cancelled agent-end lands before workflow/end', async () => {
  864. const { ctx, parent, provider } = await setup({ manual: true, config: { provider: 'stub', maxConcurrentAgents: 8, disposeGraceMs: 300 } })
  865. const ends: { seq: number; outcome: string }[] = []
  866. const order: string[] = []
  867. ctx.on('workflow/agent-start', (_info, agent) => { order.push(`start:${agent.seq}`) })
  868. ctx.on('workflow/agent-end', (_info, agent) => {
  869. ends.push({ seq: agent.seq, outcome: agent.outcome })
  870. order.push(`end:${agent.seq}`)
  871. })
  872. ctx.on('workflow/end', () => { order.push('run-end') })
  873. const handle = ctx.workflows.start({
  874. // 'slow' starts and its agent-start crosses to observers (the awaited
  875. // 'fast' call keeps the worker loop turning), then the script seizes
  876. // the loop: the wedged worker can never author slow's agent-end —
  877. // only the host's ledger can close the pair.
  878. ...scripted(`
  879. const p = agent('slow')
  880. await agent('fast')
  881. const end = Date.now() + 1500
  882. while (Date.now() < end) {}
  883. return 'raced'
  884. `),
  885. parent,
  886. })
  887. await waitFor(() => { expect(order.filter(entry => entry.startsWith('start:')).length).toBe(2) })
  888. const fast = provider.runs.find(run => (run.request.prompt[0] as { text?: string }).text === 'fast')!
  889. fast.settle(text('fast done'))
  890. handle.cancel('stop now')
  891. const result = await handle.result
  892. expect(result.stopReason).toBe('cancelled')
  893. // fast's end is the worker's own report; slow's is host-synthesized at
  894. // the force-settle — exactly one end per started seq, no third event.
  895. expect(ends).toEqual([
  896. { seq: 2, outcome: 'completed' },
  897. { seq: 1, outcome: 'cancelled' },
  898. ])
  899. // Both ends reached observers BEFORE workflow/end: a progress consumer
  900. // can finalize its state at run-end without dangling agents.
  901. expect(order.indexOf('run-end')).toBe(order.length - 1)
  902. await handle.dispose()
  903. }, 15_000)
  904. it('graceful cancellation keeps pairing worker-authored: exactly one agent-end per start, nothing synthesized on top', async () => {
  905. const { ctx, parent, provider } = await setup({ manual: true })
  906. const ends: { seq: number; outcome: string }[] = []
  907. const order: string[] = []
  908. ctx.on('workflow/agent-end', (_info, agent) => {
  909. ends.push({ seq: agent.seq, outcome: agent.outcome })
  910. order.push(`end:${agent.seq}`)
  911. })
  912. ctx.on('workflow/end', () => { order.push('run-end') })
  913. const handle = ctx.workflows.start({
  914. ...scripted("await parallel([() => agent('a'), () => agent('b')])\nreturn 'unreachable'"),
  915. parent,
  916. })
  917. await waitFor(() => { expect(provider.runs.length).toBe(2) })
  918. handle.cancel('user stop')
  919. const result = await handle.result
  920. expect(result.stopReason).toBe('cancelled')
  921. // The live worker reported both pairs itself; the ledger must not add
  922. // a synthesized duplicate on any path that settles inside the grace.
  923. expect(ends.map(end => end.outcome)).toEqual(['cancelled', 'cancelled'])
  924. expect(new Set(ends.map(end => end.seq)).size).toBe(2)
  925. expect(order.indexOf('run-end')).toBe(order.length - 1)
  926. await handle.dispose()
  927. })
  928. })
  929. describe('worker death', () => {
  930. it('the first death signal closes admission to messages Node delivers before exit', async () => {
  931. const { ctx, parent, provider } = await setup({ manual: true })
  932. const phases: string[] = []
  933. ctx.on('workflow/phase', (_info, title) => { phases.push(title) })
  934. const handle = ctx.workflows.start({
  935. ...scripted('await new Promise(() => {})'),
  936. parent,
  937. })
  938. const worker = (handle as unknown as { worker: Worker }).worker
  939. // Node may physically emit error -> queued message -> exit. Reproduce
  940. // that ordering deterministically at the Worker event boundary: the
  941. // late protocol data must not create work, narrate, or rewrite error.
  942. worker.emit('error', new Error('synthetic error-before-message'))
  943. worker.emit('message', { type: WorkerToHostType.Phase, title: 'late phase' })
  944. worker.emit('message', {
  945. type: WorkerToHostType.ChildStart,
  946. callId: 999,
  947. request: { prompt: 'late child' },
  948. })
  949. worker.emit('message', {
  950. type: WorkerToHostType.Result,
  951. result: { value: 'late', stopReason: 'completed', agentsStarted: 1 },
  952. })
  953. const result = await handle.result
  954. expect(result.stopReason).toBe('error')
  955. expect(result.error).toContain('synthetic error-before-message')
  956. expect(provider.runs).toHaveLength(0)
  957. expect(phases).toEqual([])
  958. await handle.dispose()
  959. await ctx.fiber.dispose()
  960. })
  961. it('refuses and disposes a provider run that becomes ready after its real worker dies', async () => {
  962. const ctx = new Context()
  963. await ctx.plugin(SubagentService)
  964. const requested = Promise.withResolvers<SubagentStartRequest>()
  965. const ready = Promise.withResolvers<SubagentRun>()
  966. let disposeCalls = 0
  967. const warn = vi.spyOn(ctx.logger, 'warn').mockImplementation(() => ctx.logger)
  968. const provider: SubagentProvider = {
  969. name: 'late-ready',
  970. capabilities: { outputSchema: true, depthLimit: true, toolFilter: true, persona: false },
  971. inheritsParentContext: false,
  972. start: (request) => {
  973. requested.resolve(request)
  974. // Model a backend whose independent startup boundary cannot be
  975. // interrupted promptly. The host must still reject ownership if the
  976. // worker dies before this promise transfers the ready run.
  977. return ready.promise
  978. },
  979. }
  980. ctx.subagents.registerProvider(provider)
  981. await ctx.plugin(WorkerWorkflowEngine, { provider: 'late-ready', maxConcurrentAgents: 1 })
  982. const lifecycle: string[] = []
  983. ctx.on('workflow/agent-start', () => { lifecycle.push('start') })
  984. ctx.on('workflow/agent-end', () => { lifecycle.push('end') })
  985. const handle = ctx.workflows.start({
  986. ...scripted("return await agent('pending startup')"),
  987. parent: fakeParent(),
  988. })
  989. const request = await requested.promise
  990. const worker = (handle as unknown as { worker: Worker }).worker
  991. // Kill the actual Worker while provider startup is independently
  992. // pending. Death closes admission and aborts the shared signal, but this
  993. // deliberately uncooperative provider still fulfills afterward.
  994. await worker.terminate()
  995. const result = await handle.result
  996. expect(result.stopReason).toBe('error')
  997. expect(result.error).toContain('exit code')
  998. expect(request.signal.aborted).toBe(true)
  999. expect(request.signal.reason).toBe('workflow worker gone')
  1000. ready.resolve({
  1001. id: AgentId('late-ready-child'),
  1002. result: Promise.resolve({ output: [], stopReason: 'aborted' }),
  1003. dispose: () => {
  1004. disposeCalls += 1
  1005. return Promise.reject(new Error('late ready dispose failed'))
  1006. },
  1007. })
  1008. await waitFor(() => {
  1009. expect(disposeCalls).toBe(1)
  1010. expect(warn).toHaveBeenCalledWith(expect.stringContaining('refused child dispose failed: Error: late ready dispose failed'))
  1011. }, 1000)
  1012. expect(lifecycle).toEqual([])
  1013. await handle.dispose()
  1014. expect(disposeCalls).toBe(1)
  1015. await ctx.fiber.dispose()
  1016. })
  1017. it('a worker that exits before settling reports an error result and reaps its children', async () => {
  1018. const ctx = new Context()
  1019. await ctx.plugin(SubagentService)
  1020. // The child's dispose() REJECTS on top of the worker death: the reap
  1021. // must contain it (warn, not crash) while still emptying the registry.
  1022. const signalAborts: unknown[] = []
  1023. const provider: SubagentProvider = {
  1024. name: 'doomed',
  1025. capabilities: { outputSchema: true, depthLimit: true, toolFilter: true, persona: false },
  1026. inheritsParentContext: false,
  1027. start: async (request) => {
  1028. request.signal.addEventListener('abort', () => {
  1029. signalAborts.push(request.signal.reason)
  1030. // The death claim precedes the shared-signal fanout. This
  1031. // synchronous callback cannot turn death into cancellation.
  1032. handle.cancel('reentered from worker-death signal cleanup')
  1033. }, { once: true })
  1034. return {
  1035. id: AgentId('doomed-child'),
  1036. result: new Promise(() => { /* never settles; the reap is the teardown */ }),
  1037. dispose: () => Promise.reject(new Error('dispose exploded during reap')),
  1038. }
  1039. },
  1040. }
  1041. ctx.subagents.registerProvider(provider)
  1042. await ctx.plugin(WorkerWorkflowEngine, { provider: 'doomed', maxConcurrentAgents: 2 })
  1043. const runEnds: WorkflowResultInfo[] = []
  1044. ctx.on('workflow/end', (_info, result) => { runEnds.push(result) })
  1045. const handle = ctx.workflows.start({
  1046. // The stray child's start RPC reaches the host, then the script kills
  1047. // its own worker through the documented vm escape — the host must
  1048. // settle `error` with the exit diagnostics and wind the child down.
  1049. ...scripted(`
  1050. agent('doomed')
  1051. const proc = ${ESCAPE}
  1052. const st = globalThis.constructor.constructor('return setTimeout')()
  1053. await new Promise(resolve => st(resolve, 200))
  1054. proc.exit(7)
  1055. `),
  1056. parent: fakeParent(),
  1057. })
  1058. const result = await handle.result
  1059. expect(result.stopReason).toBe('error')
  1060. expect(result.error).toContain('exit code 7')
  1061. expect(result.agentsStarted).toBe(1)
  1062. // A worker death is a stop reason like any other: workflow/end fires
  1063. // with the error outcome — for a bus observer it is the only obituary.
  1064. expect(runEnds).toEqual([{ stopReason: 'error', error: result.error, agentsStarted: 1 }])
  1065. // Result already settled — this is the reap's promptness, not a
  1066. // cold-start race; tight explicit bound (see the helper's doc comment).
  1067. await waitFor(() => {
  1068. expect(signalAborts).toEqual(['workflow worker gone'])
  1069. }, 1000)
  1070. await Promise.resolve()
  1071. expect(result.stopReason).toBe('error')
  1072. await handle.dispose()
  1073. }, 15_000)
  1074. it('an uncaught exception inside the worker surfaces as an error result and reaps the in-flight child', async () => {
  1075. const { ctx, parent, provider } = await setup({ manual: true })
  1076. const handle = ctx.workflows.start({
  1077. ...scripted(`
  1078. agent('in flight when the worker dies')
  1079. const proc = ${ESCAPE}
  1080. const st = globalThis.constructor.constructor('return setTimeout')()
  1081. await new Promise(resolve => st(resolve, 200))
  1082. proc.nextTick(() => { throw new Error('worker blew up') })
  1083. await new Promise(() => {})
  1084. `),
  1085. parent,
  1086. })
  1087. const result = await handle.result
  1088. expect(result.stopReason).toBe('error')
  1089. expect(result.error).toContain('worker blew up')
  1090. // The reap wound the stray child down (cancel + a CLEAN dispose).
  1091. // Result already settled — this is the reap's promptness, not a
  1092. // cold-start race; tight explicit bound (see the helper's doc comment).
  1093. await waitFor(() => {
  1094. expect(provider.runs.length).toBe(1)
  1095. expect(provider.runs[0]!.disposed).toBe(true)
  1096. }, 1000)
  1097. await handle.dispose()
  1098. }, 15_000)
  1099. it('a worker death pairs every stranded start: the synthesized cancelled agent-end precedes the error workflow/end', async () => {
  1100. const { ctx, parent, provider } = await setup({ manual: true })
  1101. const ends: { seq: number; outcome: string }[] = []
  1102. const order: string[] = []
  1103. ctx.on('workflow/agent-start', (_info, agent) => { order.push(`start:${agent.seq}`) })
  1104. ctx.on('workflow/agent-end', (_info, agent) => {
  1105. ends.push({ seq: agent.seq, outcome: agent.outcome })
  1106. order.push(`end:${agent.seq}`)
  1107. })
  1108. ctx.on('workflow/end', () => { order.push('run-end') })
  1109. const handle = ctx.workflows.start({
  1110. // Same choreography as the force-settle pairing test, but the worker
  1111. // DIES (the documented vm escape) instead of being terminated: the
  1112. // exit path must close slow's pair from the ledger too. The escaped
  1113. // setTimeout lets the already-posted messages flush before the kill.
  1114. ...scripted(`
  1115. const p = agent('slow')
  1116. await agent('fast')
  1117. const proc = ${ESCAPE}
  1118. const st = globalThis.constructor.constructor('return setTimeout')()
  1119. await new Promise(resolve => st(resolve, 150))
  1120. proc.exit(7)
  1121. `),
  1122. parent,
  1123. })
  1124. await waitFor(() => { expect(order.filter(entry => entry.startsWith('start:')).length).toBe(2) })
  1125. const fast = provider.runs.find(run => (run.request.prompt[0] as { text?: string }).text === 'fast')!
  1126. fast.settle(text('fast done'))
  1127. const result = await handle.result
  1128. expect(result.stopReason).toBe('error')
  1129. expect(result.error).toContain('exit code 7')
  1130. expect(ends).toEqual([
  1131. { seq: 2, outcome: 'completed' },
  1132. { seq: 1, outcome: 'cancelled' },
  1133. ])
  1134. expect(order.indexOf('run-end')).toBe(order.length - 1)
  1135. await handle.dispose()
  1136. }, 15_000)
  1137. it('a dispose ack racing the worker death is dropped, not crashed (post after exit)', async () => {
  1138. // Slow child disposal: the ack resolves only AFTER the worker died, so
  1139. // it has nowhere to go and must be dropped silently (the workerGone
  1140. // guard in post()).
  1141. const { ctx, parent, provider } = await setup({ disposeDelayMs: 300 })
  1142. const handle = ctx.workflows.start({
  1143. // The STRAY child settles instantly, so its wrapper starts the slow
  1144. // host-side disposal concurrently while the script goes on to kill
  1145. // its own worker — the ack then resolves into a dead thread.
  1146. ...scripted(`
  1147. agent('stray, never awaited')
  1148. const proc = ${ESCAPE}
  1149. const st = globalThis.constructor.constructor('return setTimeout')()
  1150. await new Promise(resolve => st(resolve, 150))
  1151. proc.exit(5)
  1152. `),
  1153. parent,
  1154. })
  1155. const result = await handle.result
  1156. expect(result.stopReason).toBe('error')
  1157. expect(result.error).toContain('exit code 5')
  1158. // Result already settled — this is the reap's promptness (bounded
  1159. // above the mock's fixed 300ms dispose delay, not a cold-start race);
  1160. // tight explicit bound (see the helper's doc comment).
  1161. await waitFor(() => { expect(provider.runs[0]!.disposed).toBe(true) }, 1000)
  1162. await handle.dispose()
  1163. }, 15_000)
  1164. it('a worker death AFTER a cancel reports cancelled, not error', async () => {
  1165. const { ctx, parent } = await setup({ config: { provider: 'stub', disposeGraceMs: 60_000 } })
  1166. const handle = ctx.workflows.start({
  1167. ...scripted(`
  1168. const proc = ${ESCAPE}
  1169. const st = globalThis.constructor.constructor('return setTimeout')()
  1170. log('armed')
  1171. await new Promise(resolve => st(resolve, 400))
  1172. proc.exit(3)
  1173. `),
  1174. parent,
  1175. })
  1176. const logs: string[] = []
  1177. ctx.on('workflow/log', (_info, message) => { logs.push(message) })
  1178. await waitFor(() => { expect(logs).toContain('armed') })
  1179. handle.cancel('stop it')
  1180. // The grace is deliberately huge: only the worker's own death (exit 3,
  1181. // unreachable by the cancel — the script ignores hooks) settles this.
  1182. const result = await handle.result
  1183. expect(result.stopReason).toBe('cancelled')
  1184. expect(result.error).toContain('stop it')
  1185. await handle.dispose()
  1186. }, 15_000)
  1187. })
  1188. describe('service surface', () => {
  1189. it('run ids are unique and lifecycle meta is the run\'s borrowed immutable value', async () => {
  1190. const { ctx, parent } = await setup()
  1191. let eventMeta: WorkflowRunInfo | undefined
  1192. ctx.on('workflow/start', (info) => { eventMeta = info })
  1193. const first = ctx.workflows.start({ ...scripted('return 1'), parent })
  1194. const second = ctx.workflows.start({ ...scripted('return 2'), parent })
  1195. expect(first.id).not.toBe(second.id)
  1196. expect(eventMeta!.meta).toBe(second.meta)
  1197. expect(second.meta.name).toBe('test-flow')
  1198. await Promise.all([first.result, second.result])
  1199. await first.dispose()
  1200. await second.dispose()
  1201. })
  1202. it('unregisters ctx.workflows when the engine fiber is disposed (HMR safety)', async () => {
  1203. const ctx = new Context()
  1204. await ctx.plugin(SubagentService)
  1205. const fiber = await ctx.plugin(WorkerWorkflowEngine, {})
  1206. expect(ctx.get('workflows')).toBeDefined()
  1207. await fiber.dispose()
  1208. expect(ctx.get('workflows')).toBeUndefined()
  1209. })
  1210. it('keeps a holder-owned run usable when the engine unloads before its child starts', async () => {
  1211. const { ctx, parent, provider, engineFiber } = await setup({ reply: () => text('survived reload') })
  1212. let handle!: ReturnType<typeof ctx.workflows.start>
  1213. const holder = await ctx.plugin(Object.assign((inner: Context) => {
  1214. handle = inner.workflows.start({ ...scripted("return await agent('after reload')"), parent })
  1215. }, { inject: ['workflows'] }))
  1216. try {
  1217. // A real worker cannot deliver child-start in the synchronous start()
  1218. // slice. Unload the provider before that message arrives: the returned
  1219. // run belongs to `holder`, not to the engine fiber being reloaded.
  1220. expect(provider.runs).toHaveLength(0)
  1221. await engineFiber.dispose()
  1222. expect(ctx.get('workflows')).toBeUndefined()
  1223. await expect(handle.result).resolves.toEqual({
  1224. value: 'survived reload',
  1225. stopReason: 'completed',
  1226. agentsStarted: 1,
  1227. })
  1228. expect(provider.runs).toHaveLength(1)
  1229. } finally {
  1230. await handle.dispose()
  1231. await holder.dispose()
  1232. await ctx.fiber.dispose()
  1233. }
  1234. })
  1235. it('has the class-plugin export shape (default = the engine service class)', () => {
  1236. expect(workerEngineModule.default).toBe(WorkerWorkflowEngine)
  1237. const loader = Object.create(Loader.prototype) as Loader
  1238. const unwrapped: unknown = loader.unwrapExports(workerEngineModule)
  1239. expect(unwrapped).toBe(WorkerWorkflowEngine)
  1240. })
  1241. })
  1242. })