workflow-workerthread.spec.ts 64 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372
  1. import { describe, expect, it, vi } from 'vitest'
  2. import { fileURLToPath } from 'node:url'
  3. import type { Worker } from 'node:worker_threads'
  4. import { Context } from 'cordis'
  5. import Loader from '@cordisjs/plugin-loader'
  6. import type { Agent } from '@deepseek-ai/dsh-agent'
  7. import SubagentService from '@deepseek-ai/dsh-subagent'
  8. import type { SubagentCapabilities, SubagentProvider, SubagentResult, SubagentRun, SubagentStartRequest } from '@deepseek-ai/dsh-subagent'
  9. import type { WorkflowMeta, WorkflowResult, WorkflowResultInfo, WorkflowRunInfo } from '@deepseek-ai/dsh-workflow'
  10. import * as workerEngineModule from '../src/index.ts'
  11. import WorkerWorkflowEngine, { type Config } from '../src/index.ts'
  12. import { HostToWorkerType, WorkerToHostType } from '../src/protocol.ts'
  13. import { SessionId } from '@deepseek-ai/dsh-session'
  14. /** A minimal parent stand-in: the engine only threads it through to the provider. */
  15. function fakeParent(): Agent {
  16. return { id: SessionId('workflow-parent'), options: {} } as unknown as Agent
  17. }
  18. // Allow cold worker startup on contended CI runners.
  19. vi.setConfig({ testTimeout: 30_000 })
  20. /**
  21. * Wait up to 10 seconds for CPU-bound worker startup or cross-thread delivery on contended CI.
  22. * Host reactions after an observed event use explicit tight overrides, so this generous startup
  23. * allowance cannot hide multi-second reap regressions.
  24. */
  25. function waitFor(assertion: () => void, timeout = 10_000): Promise<void> {
  26. return vi.waitFor(assertion, { timeout, interval: 50 })
  27. }
  28. /** The vm-context escape hatch, spelled once: real Worker tests use it to make the WORKER misbehave. */
  29. const ESCAPE = "globalThis.constructor.constructor('return process')()"
  30. /** One controllable child run: the test (or auto mode) settles it. */
  31. interface ControlledRun {
  32. request: SubagentStartRequest
  33. /** Fulfill the provider's async start with a ready child. */
  34. publish(): void
  35. /** Reject the provider's async start before ownership transfer. */
  36. rejectStart(error: unknown): void
  37. settle(result: SubagentResult): void
  38. rejectResult(error: unknown): void
  39. cancelled: string | undefined
  40. disposed: boolean
  41. disposeCalls: number
  42. }
  43. /**
  44. * A scripted in-test provider over the REAL SubagentService registry: `auto`
  45. * settles each run via the reply function on a microtask; `manual` piles runs
  46. * up in `runs` for the test to settle. A run aborts (settles `aborted`) when
  47. * the request signal fires, like the real in-process backends.
  48. */
  49. class StubProvider implements SubagentProvider {
  50. readonly capabilities: SubagentCapabilities = { outputSchema: true, depthLimit: true, toolFilter: true, persona: false }
  51. readonly inheritsParentContext = false
  52. readonly runs: ControlledRun[] = []
  53. constructor(
  54. readonly name: string,
  55. private readonly reply?: (request: SubagentStartRequest, index: number) => SubagentResult,
  56. private readonly disposeDelayMs = 0,
  57. private readonly deferStart = false,
  58. private readonly onAbortString?: (reason: string | undefined, index: number) => void,
  59. private readonly onSignalAbort?: (reason: unknown, index: number) => void,
  60. ) {}
  61. async start(request: SubagentStartRequest): Promise<SubagentRun> {
  62. const startGate = Promise.withResolvers<undefined>()
  63. const terminal = Promise.withResolvers<SubagentResult>()
  64. terminal.promise.catch(() => { /* provider owns early settlement until publication */ })
  65. let published = false
  66. const controlled: ControlledRun = {
  67. request,
  68. publish: () => { published = true; startGate.resolve(undefined) },
  69. rejectStart: (error) => { startGate.reject(error) },
  70. settle: (result) => { terminal.resolve(result) },
  71. rejectResult: (error) => { terminal.reject(error) },
  72. cancelled: undefined,
  73. disposed: false,
  74. disposeCalls: 0,
  75. }
  76. this.runs.push(controlled)
  77. const index = this.runs.length - 1
  78. request.signal.addEventListener('abort', () => {
  79. controlled.cancelled = String(request.signal.reason ?? 'cancelled')
  80. this.onAbortString?.(String(request.signal.reason ?? 'cancelled'), index)
  81. this.onSignalAbort?.(request.signal.reason, index)
  82. if (published) terminal.resolve({ output: [], stopReason: 'aborted' })
  83. else startGate.reject(new Error('child start aborted before publication'))
  84. }, { once: true })
  85. if (!this.deferStart) controlled.publish()
  86. if (this.reply) {
  87. const reply = this.reply
  88. queueMicrotask(() => { terminal.resolve(reply(request, index)) })
  89. }
  90. try {
  91. await startGate.promise
  92. } catch (error: unknown) {
  93. controlled.disposeCalls += 1
  94. controlled.disposed = true
  95. throw error
  96. }
  97. if (request.signal.aborted) throw new Error('child start aborted before publication')
  98. return {
  99. id: SessionId(`stub-child-${index}`),
  100. localAgent: undefined,
  101. result: terminal.promise,
  102. dispose: () => {
  103. controlled.disposeCalls += 1
  104. if (this.disposeDelayMs === 0) {
  105. controlled.disposed = true
  106. return Promise.resolve()
  107. }
  108. return new Promise<void>((resolve) => {
  109. setTimeout(() => {
  110. controlled.disposed = true
  111. resolve()
  112. }, this.disposeDelayMs)
  113. })
  114. },
  115. }
  116. }
  117. }
  118. /** Text-reply helper for auto providers. */
  119. function text(reply: string): SubagentResult {
  120. return { output: [{ type: 'text', text: reply }], stopReason: 'completed' }
  121. }
  122. interface SetupOptions {
  123. config?: Config
  124. reply?: (request: SubagentStartRequest, index: number) => SubagentResult
  125. manual?: boolean
  126. disposeDelayMs?: number
  127. deferStart?: boolean
  128. onChildAbortString?: (reason: string | undefined, index: number) => void
  129. onChildSignalAbort?: (reason: unknown, index: number) => void
  130. }
  131. async function setup(options?: SetupOptions) {
  132. const ctx = new Context()
  133. await ctx.plugin(SubagentService)
  134. const provider = new StubProvider(
  135. 'stub',
  136. options?.manual ? undefined : options?.reply ?? (() => text('stub reply')),
  137. options?.disposeDelayMs ?? 0,
  138. options?.deferStart ?? false,
  139. options?.onChildAbortString,
  140. options?.onChildSignalAbort,
  141. )
  142. ctx.subagents.registerProvider(provider)
  143. // A fixed concurrency ceiling: the auto-resolved default is machine-derived
  144. // (cores - 2, floored at 1), so tests that expect N children in flight
  145. // would wedge on small CI runners.
  146. const engineFiber = await ctx.plugin(WorkerWorkflowEngine, { provider: 'stub', maxConcurrentAgents: 8, ...options?.config })
  147. return { ctx, provider, parent: fakeParent(), engineFiber }
  148. }
  149. /** The standard test meta plus a body, spread into a start request. */
  150. function scripted(body: string, metaExtra?: Partial<WorkflowMeta>): { script: string; meta: WorkflowMeta } {
  151. return { script: body, meta: { name: 'test-flow', description: 'a test workflow', ...metaExtra } }
  152. }
  153. /** Start + await one run, disposing on the way out. */
  154. async function run(ctx: Context, parent: Agent, source: { script: string; meta: WorkflowMeta }, args?: unknown): Promise<WorkflowResult> {
  155. const handle = ctx.workflows.start({ ...source, parent, ...args !== undefined ? { args } : {} })
  156. try {
  157. return await handle.result
  158. } finally {
  159. await handle.dispose()
  160. }
  161. }
  162. describe('dsh-workflow-workerthread', () => {
  163. describe('script execution over a real worker thread', () => {
  164. it('runs a script end-to-end: agent() text results, phases, log, args, return value, events', async () => {
  165. const { ctx, parent, provider } = await setup({ reply: (_request, index) => text(`answer-${index}`) })
  166. const events: [string, unknown[]][] = []
  167. for (const name of ['workflow/start', 'workflow/phase', 'workflow/log', 'workflow/agent-start', 'workflow/agent-end', 'workflow/end'] as const) {
  168. ctx.on(name, (...payload: unknown[]) => { events.push([name, payload]) })
  169. }
  170. const result = await run(ctx, parent, scripted(`
  171. phase('Scan')
  172. log('starting with ' + args.files.length + ' files')
  173. const answers = await pipeline(args.files, (prev, item) => agent('read ' + item))
  174. phase('Report')
  175. return { answers, count: args.files.length }
  176. `, { phases: [{ title: 'Scan' }, { title: 'Report' }] }), { files: ['a.ts', 'b.ts'] })
  177. expect(result.stopReason).toBe('completed')
  178. expect(result.agentsStarted).toBe(2)
  179. expect(result.value).toEqual({ answers: ['answer-0', 'answer-1'], count: 2 })
  180. expect(provider.runs.every(r => r.disposed)).toBe(true)
  181. const names = events.map(([name]) => name)
  182. expect(names[0]).toBe('workflow/start')
  183. expect(names).toContain('workflow/phase')
  184. expect(names).toContain('workflow/log')
  185. expect(names.at(-1)).toBe('workflow/end')
  186. const info = events[0]![1][0] as WorkflowRunInfo
  187. expect(info.meta.name).toBe('test-flow')
  188. const end = events.at(-1)![1][1] as Record<string, unknown>
  189. expect(end).toEqual({ stopReason: 'completed', agentsStarted: 2 })
  190. expect('value' in end).toBe(false)
  191. })
  192. it('agent({schema, model}) forwards outputSchema and agentOptions to the provider across the thread', async () => {
  193. const { ctx, parent, provider } = await setup({
  194. reply: () => ({ output: [], structured: { files: ['x.ts', 'y.ts'] }, stopReason: 'completed' }),
  195. })
  196. const result = await run(ctx, parent, scripted(`
  197. const found = await agent('list files', { model: 'deepseek-v4-pro', schema: { type: 'object', properties: { files: { type: 'array', items: { type: 'string' } } }, required: ['files'] } })
  198. return { first: found.files[0], count: found.files.length }
  199. `))
  200. expect(result.value).toEqual({ first: 'x.ts', count: 2 })
  201. expect(provider.runs[0]!.request.outputSchema).toEqual({
  202. type: 'object',
  203. properties: { files: { type: 'array', items: { type: 'string' } } },
  204. required: ['files'],
  205. })
  206. expect(provider.runs[0]!.request.agentOptions).toEqual({ model: 'deepseek-v4-pro' })
  207. expect(provider.runs[0]!.request.parent).toBeDefined()
  208. })
  209. it('agent({provider}) forwards provider-only agentOptions across the thread', async () => {
  210. const { ctx, parent, provider } = await setup()
  211. const result = await run(ctx, parent, scripted("return await agent('route me', { provider: 'openai' })"))
  212. expect(result.value).toBe('stub reply')
  213. expect(provider.runs[0]!.request.agentOptions).toEqual({ provider: 'openai' })
  214. })
  215. it('a start-request provider override selects every child without changing the engine default', async () => {
  216. const { ctx, parent, provider } = await setup()
  217. const selected = new StubProvider('selected', () => text('selected reply'))
  218. ctx.subagents.registerProvider(selected)
  219. const overridden = ctx.workflows.start({
  220. ...scripted("return await agent('route this run')"),
  221. parent,
  222. subagentProvider: 'selected',
  223. })
  224. expect((await overridden.result).value).toBe('selected reply')
  225. await overridden.dispose()
  226. expect(selected.runs).toHaveLength(1)
  227. expect(provider.runs).toHaveLength(0)
  228. const ordinary = await run(ctx, parent, scripted("return await agent('use the default')"))
  229. expect(ordinary.value).toBe('stub reply')
  230. expect(provider.runs).toHaveLength(1)
  231. })
  232. it('a fatal hook error inside the worker kills the script and reports the error', async () => {
  233. const { ctx, parent } = await setup()
  234. const result = await run(ctx, parent, scripted("return await parallel([() => agent('x', { isolation: 'worktree' })])"))
  235. expect(result.stopReason).toBe('error')
  236. expect(result.error).toContain('"isolation" is deferred')
  237. })
  238. it('a provider start failure crosses back as a fatal AGENT_START error', async () => {
  239. const { ctx, parent } = await setup({ config: { provider: 'nonexistent' } })
  240. const result = await run(ctx, parent, scripted("return await pipeline([1], () => agent('p'))"))
  241. expect(result.stopReason).toBe('error')
  242. expect(result.error).toContain('agent() could not start a child')
  243. })
  244. it('waits for async provider start before announcing a result that settled early', async () => {
  245. const { ctx, parent, provider } = await setup({ manual: true, deferStart: true })
  246. const order: string[] = []
  247. ctx.on('workflow/agent-start', (_info, agent) => { order.push(`start:${agent.seq}`) })
  248. ctx.on('workflow/agent-end', (_info, agent) => { order.push(`end:${agent.outcome}`) })
  249. ctx.on('workflow/end', () => { order.push('run-end') })
  250. const handle = ctx.workflows.start({ ...scripted("return await agent('p')"), parent })
  251. await waitFor(() => { expect(provider.runs.length).toBe(1) })
  252. const early = text('accepted value')
  253. provider.runs[0]!.settle(early)
  254. // The provider still owns this early result while start is pending.
  255. await new Promise(resolve => setTimeout(resolve, 0))
  256. expect(order).toEqual([])
  257. provider.runs[0]!.publish()
  258. const result = await handle.result
  259. expect(result.value).toBe('accepted value')
  260. expect(order).toEqual(['start:1', 'end:completed', 'run-end'])
  261. await handle.dispose()
  262. expect(provider.runs[0]!.disposeCalls).toBe(1)
  263. })
  264. it('observes an early result rejection but sends ChildStarted before ChildFailed after start fulfills', async () => {
  265. const { ctx, parent, provider } = await setup({ manual: true, deferStart: true })
  266. const lifecycle: string[] = []
  267. ctx.on('workflow/agent-start', () => { lifecycle.push('start') })
  268. ctx.on('workflow/agent-end', (_info, agent) => { lifecycle.push(`end:${agent.outcome}`) })
  269. const handle = ctx.workflows.start({
  270. ...scripted("try { await agent('p'); return 'unreachable' } catch (e) { return { code: e.code, message: e.message } }"),
  271. parent,
  272. })
  273. const worker = (handle as unknown as { worker: { postMessage(message: unknown): void } }).worker
  274. const post = vi.spyOn(worker, 'postMessage')
  275. const childMessageTypes = (): HostToWorkerType[] => post.mock.calls
  276. .map(([message]) => (message as { type: HostToWorkerType }).type)
  277. .filter(type => type === HostToWorkerType.ChildStarted || type === HostToWorkerType.ChildFailed)
  278. await waitFor(() => { expect(provider.runs.length).toBe(1) })
  279. provider.runs[0]!.rejectResult(new Error('backend failed before publication'))
  280. await new Promise(resolve => setTimeout(resolve, 0))
  281. expect(childMessageTypes()).toEqual([])
  282. expect(lifecycle).toEqual([])
  283. provider.runs[0]!.publish()
  284. const result = await handle.result
  285. expect(result.value).toMatchObject({ code: 'AGENT_RESULT' })
  286. expect((result.value as { message: string }).message).toContain('backend failed before publication')
  287. expect(childMessageTypes()).toEqual([HostToWorkerType.ChildStarted, HostToWorkerType.ChildFailed])
  288. expect(lifecycle).toEqual(['start', 'end:failed'])
  289. post.mockRestore()
  290. await handle.dispose()
  291. })
  292. it('classifies provider start rejection as AGENT_START, drops an early result, and emits no false lifecycle pair', async () => {
  293. const { ctx, parent, provider } = await setup({ manual: true, deferStart: true })
  294. const lifecycle: string[] = []
  295. ctx.on('workflow/agent-start', () => { lifecycle.push('start') })
  296. ctx.on('workflow/agent-end', () => { lifecycle.push('end') })
  297. const handle = ctx.workflows.start({
  298. ...scripted("try { await agent('p'); return 'unreachable' } catch (e) { return { code: e.code, message: e.message } }"),
  299. parent,
  300. })
  301. await waitFor(() => { expect(provider.runs.length).toBe(1) })
  302. // ACP-style failure can settle result(error) before its session/publication
  303. // boundary rejects. Start rejection must dominate that buffered child outcome.
  304. provider.runs[0]!.settle({ output: [], stopReason: 'error' })
  305. await new Promise(resolve => setTimeout(resolve, 0))
  306. provider.runs[0]!.rejectStart(new Error('publication rolled back'))
  307. const result = await handle.result
  308. expect(result.value).toMatchObject({ code: 'AGENT_START' })
  309. expect((result.value as { message: string }).message).toContain('publication rolled back')
  310. expect(lifecycle).toEqual([])
  311. await waitFor(() => {
  312. expect(provider.runs[0]!.disposed).toBe(true)
  313. expect(provider.runs[0]!.disposeCalls).toBe(1)
  314. })
  315. await handle.dispose()
  316. expect(provider.runs[0]!.disposeCalls).toBe(1)
  317. })
  318. it('aborts a pending provider start once without publishing workflow lifecycle', async () => {
  319. const { ctx, parent, provider } = await setup({ manual: true, deferStart: true, config: { disposeGraceMs: 500 } })
  320. const lifecycle: string[] = []
  321. ctx.on('workflow/agent-start', () => { lifecycle.push('start') })
  322. ctx.on('workflow/agent-end', () => { lifecycle.push('end') })
  323. const handle = ctx.workflows.start({ ...scripted("return await agent('pending')"), parent })
  324. await waitFor(() => { expect(provider.runs.length).toBe(1) })
  325. const disposal = handle.dispose()
  326. await waitFor(() => {
  327. expect(provider.runs[0]!.cancelled).toBe('workflow disposed')
  328. expect(provider.runs[0]!.disposed).toBe(true)
  329. })
  330. // Ensure the host-driven disposal removed the registry entry before the
  331. // late start rejection; its callback must not invoke dispose again.
  332. await new Promise(resolve => setTimeout(resolve, 0))
  333. provider.runs[0]!.rejectStart(new Error('cancelled before publication'))
  334. const result = await handle.result
  335. await disposal
  336. expect(result.stopReason).toBe('cancelled')
  337. expect(lifecycle).toEqual([])
  338. expect(provider.runs[0]!.disposeCalls).toBe(1)
  339. })
  340. it('a child result REJECTION crosses back as a fatal AGENT_RESULT error (a broken provider is not a failed child)', async () => {
  341. const ctx = new Context()
  342. await ctx.plugin(SubagentService)
  343. const provider: SubagentProvider = {
  344. name: 'rejecting',
  345. capabilities: { outputSchema: true, depthLimit: true, toolFilter: true, persona: false },
  346. inheritsParentContext: false,
  347. start: async () => ({
  348. id: SessionId('reject-child'),
  349. localAgent: undefined,
  350. result: Promise.reject(new Error('backend exploded')),
  351. dispose: () => Promise.resolve(),
  352. }),
  353. }
  354. ctx.subagents.registerProvider(provider)
  355. await ctx.plugin(WorkerWorkflowEngine, { provider: 'rejecting', maxConcurrentAgents: 2 })
  356. const result = await run(ctx, fakeParent(), scripted(`
  357. try { await agent('p'); return 'unreachable' } catch (e) { return { name: e.name, code: e.code, fatal: e.fatal, message: e.message } }
  358. `))
  359. expect(result.value).toMatchObject({ name: 'WorkflowError', code: 'AGENT_RESULT', fatal: true })
  360. expect((result.value as { message: string }).message).toContain('backend exploded')
  361. })
  362. it('maps a non-JSON ready-child result to fatal AGENT_RESULT instead of wedging the bridge', async () => {
  363. const { ctx, parent } = await setup({
  364. reply: () => ({ output: [], structured: () => { /* deliberately outside lossless JSON */ }, stopReason: 'completed' }),
  365. })
  366. const result = await run(ctx, parent, scripted(`
  367. try { await agent('p'); return 'unreachable' } catch (e) { return { code: e.code, message: e.message } }
  368. `))
  369. expect(result.value).toMatchObject({ code: 'AGENT_RESULT' })
  370. expect((result.value as { message: string }).message).toContain('workflow child result could not cross the worker boundary')
  371. })
  372. it('contains a non-JSON result even if the injected subagent service violates its normalization contract', async () => {
  373. // The real worker boundary must reject a non-JSON same-process result.
  374. const { ctx, parent } = await setup()
  375. const invalid = {
  376. output: [],
  377. structured: () => { /* deliberately outside lossless JSON */ },
  378. stopReason: 'completed',
  379. } as unknown as SubagentResult
  380. const start = vi.spyOn(ctx.subagents, 'start').mockResolvedValue({
  381. id: SessionId('raw-invalid-child'),
  382. localAgent: undefined,
  383. result: Promise.resolve(invalid),
  384. dispose: () => Promise.resolve(),
  385. })
  386. const result = await run(ctx, parent, scripted(`
  387. try { await agent('p'); return 'unreachable' } catch (e) { return { code: e.code, message: e.message } }
  388. `))
  389. expect(start).toHaveBeenCalledOnce()
  390. expect(result.value).toMatchObject({ code: 'AGENT_RESULT' })
  391. expect((result.value as { message: string }).message)
  392. .toContain('workflow child result could not cross the worker boundary')
  393. })
  394. it('a child whose dispose() throws synchronously cannot wedge the script (the host acks anyway)', async () => {
  395. const ctx = new Context()
  396. await ctx.plugin(SubagentService)
  397. const provider: SubagentProvider = {
  398. name: 'bad-dispose',
  399. capabilities: { outputSchema: true, depthLimit: true, toolFilter: true, persona: false },
  400. inheritsParentContext: false,
  401. start: async () => ({
  402. id: SessionId('bad-dispose-child'),
  403. localAgent: undefined,
  404. result: Promise.resolve({ output: [{ type: 'text', text: 'fine' }], stopReason: 'completed' }),
  405. cancel: () => { /* settled already */ },
  406. dispose: () => { throw new Error('dispose exploded') },
  407. }),
  408. }
  409. ctx.subagents.registerProvider(provider)
  410. await ctx.plugin(WorkerWorkflowEngine, { provider: 'bad-dispose', maxConcurrentAgents: 2 })
  411. const result = await run(ctx, fakeParent(), scripted("return await agent('p')"))
  412. expect(result.stopReason).toBe('completed')
  413. expect(result.value).toBe('fine')
  414. })
  415. it('a child dispose() rejecting an UNRENDERABLE value still acks — the containment warn is total', async () => {
  416. const ctx = new Context()
  417. await ctx.plugin(SubagentService)
  418. const provider: SubagentProvider = {
  419. name: 'coercion-trap-dispose',
  420. capabilities: { outputSchema: true, depthLimit: true, toolFilter: true, persona: false },
  421. inheritsParentContext: false,
  422. start: async () => ({
  423. id: SessionId('trap-child'),
  424. localAgent: undefined,
  425. result: Promise.resolve({ output: [{ type: 'text', text: 'fine' }], stopReason: 'completed' }),
  426. cancel: () => { /* settled already */ },
  427. // The rejection VALUE's own coercion throws: a warn built with bare
  428. // String(error) would itself throw, skipping the ChildDisposed ack
  429. // and wedging the script's finally until the grace/terminate path.
  430. // eslint-disable-next-line @typescript-eslint/prefer-promise-reject-errors -- the non-Error rejection IS the scenario under test
  431. dispose: () => Promise.reject({ toString: () => { throw new Error('coercion trap') } }),
  432. }),
  433. }
  434. ctx.subagents.registerProvider(provider)
  435. await ctx.plugin(WorkerWorkflowEngine, { provider: 'coercion-trap-dispose', maxConcurrentAgents: 2 })
  436. const result = await run(ctx, fakeParent(), scripted("return await agent('p')"))
  437. expect(result.stopReason).toBe('completed')
  438. expect(result.value).toBe('fine')
  439. })
  440. it('the worker spawns with an EMPTY environment: an escaped script finds no ambient credentials', async () => {
  441. const { ctx, parent } = await setup()
  442. // A canary in the HARNESS process's env: with an inherited environment
  443. // the escape below would read it back (exactly how DEEPSEEK_API_KEY
  444. // would leak); env: {} in the spawn options is what keeps it out.
  445. process.env.WORKFLOW_ENV_CANARY = 'leak me'
  446. try {
  447. const result = await run(ctx, parent, scripted(`
  448. const proc = ${ESCAPE}
  449. return { canary: proc.env.WORKFLOW_ENV_CANARY ?? null, keys: Object.keys(proc.env).length }
  450. `))
  451. expect(result.stopReason).toBe('completed')
  452. expect(result.value).toEqual({ canary: null, keys: 0 })
  453. } finally {
  454. delete process.env.WORKFLOW_ENV_CANARY
  455. }
  456. })
  457. it('the unbuilt worker forwards exactly TSX_TSCONFIG_PATH through the scrub: the paths-map pin survives, secrets do not', async () => {
  458. const { ctx, parent } = await setup()
  459. // The ACP snapshot harness runs the parent with its cwd OUTSIDE the
  460. // repo and pins the repo tsconfig through this variable; the worker
  461. // must inherit the pin (or its dsh-* imports silently resolve to
  462. // unbuilt lib/ bundles) while every other variable stays scrubbed.
  463. const tsconfig = fileURLToPath(new URL('../../../../tsconfig.json', import.meta.url))
  464. process.env.TSX_TSCONFIG_PATH = tsconfig
  465. process.env.WORKFLOW_ENV_CANARY = 'leak me'
  466. try {
  467. const result = await run(ctx, parent, scripted(`
  468. const proc = ${ESCAPE}
  469. return { keys: Object.keys(proc.env), tsconfig: proc.env.TSX_TSCONFIG_PATH }
  470. `))
  471. expect(result.stopReason).toBe('completed')
  472. expect(result.value).toEqual({ keys: ['TSX_TSCONFIG_PATH'], tsconfig })
  473. } finally {
  474. delete process.env.TSX_TSCONFIG_PATH
  475. delete process.env.WORKFLOW_ENV_CANARY
  476. }
  477. })
  478. })
  479. describe('lifecycle: parse errors, cancellation, termination, disposal', () => {
  480. it('start() throws synchronously for invalid meta data or an unparseable body (host-side pre-checks)', async () => {
  481. const { ctx, parent } = await setup()
  482. // Meta is DATA — shape violations reject loud, every one named.
  483. expect(() => ctx.workflows.start({ script: 'return 1', meta: { name: '', description: 'd' }, parent })).toThrow(/meta\.name must be a non-empty string/)
  484. expect(() => ctx.workflows.start({ script: 'return 1', meta: { name: 'x', description: 'd', extra: 1 } as unknown as WorkflowMeta, parent })).toThrow(/META_INVALID|not a recognized field/)
  485. expect(() => ctx.workflows.start({ ...scripted('return ((('), parent })).toThrow(/does not parse/)
  486. // The likeliest authoring slip — a Claude Code-style meta header in the
  487. // body — gets a pointed message, not a bare SyntaxError.
  488. expect(() => ctx.workflows.start({ ...scripted("export const meta = { name: 'x', description: 'd' }\nreturn 1"), parent })).toThrow(/meta rides the `meta` request field/)
  489. })
  490. it('cancel() aborts in-flight children (signal AND cancel RPC) and settles the run cancelled', async () => {
  491. const { ctx, parent, provider } = await setup({ manual: true })
  492. const ends: unknown[] = []
  493. ctx.on('workflow/agent-end', (_info, agent) => { ends.push(agent) })
  494. const runEnds: WorkflowResultInfo[] = []
  495. ctx.on('workflow/end', (_info, result) => { runEnds.push(result) })
  496. const handle = ctx.workflows.start({ ...scripted("return await agent('long job')"), parent })
  497. await waitFor(() => { expect(provider.runs.length).toBe(1) })
  498. handle.cancel('user stopped it')
  499. const result = await handle.result
  500. expect(result.stopReason).toBe('cancelled')
  501. expect(result.error).toContain('user stopped it')
  502. await handle.dispose()
  503. expect(provider.runs[0]!.disposed).toBe(true)
  504. expect(ends).toEqual([expect.objectContaining({ seq: 1, outcome: 'cancelled' })])
  505. // workflow/end is an observer's only death signal: it fires for a
  506. // cancelled run too, mirroring the settled outcome data.
  507. expect(runEnds).toEqual([{ stopReason: 'cancelled', error: result.error, agentsStarted: result.agentsStarted }])
  508. })
  509. it('an already-aborted request signal cancels before the body ever runs (the go handshake holds it)', async () => {
  510. const { ctx, parent, provider } = await setup()
  511. const controller = new AbortController()
  512. controller.abort()
  513. const logs: string[] = []
  514. ctx.on('workflow/log', (_info, message) => { logs.push(message) })
  515. const handle = ctx.workflows.start({ ...scripted("log('ran')\nreturn 123"), parent, signal: controller.signal })
  516. const result = await handle.result
  517. expect(result.stopReason).toBe('cancelled')
  518. expect(result.value).toBeNull()
  519. expect(logs).toEqual([])
  520. expect(provider.runs.length).toBe(0)
  521. await handle.dispose()
  522. })
  523. it('cancel() right after start() cancels before the body runs; the signal aborting mid-run cancels like cancel()', async () => {
  524. const { ctx, parent, provider } = await setup({ manual: true })
  525. const first = ctx.workflows.start({ ...scripted("return await agent('never')"), parent })
  526. // No-reason cancel: the canonical default reason must ride the result.
  527. first.cancel()
  528. const firstResult = await first.result
  529. expect(firstResult.stopReason).toBe('cancelled')
  530. expect(firstResult.error).toContain('workflow cancelled')
  531. expect(provider.runs.length).toBe(0)
  532. await first.dispose()
  533. const controller = new AbortController()
  534. const second = ctx.workflows.start({ ...scripted("return await agent('job')"), parent, signal: controller.signal })
  535. await waitFor(() => { expect(provider.runs.length).toBe(1) })
  536. controller.abort()
  537. expect((await second.result).stopReason).toBe('cancelled')
  538. await second.dispose()
  539. })
  540. it('removes the exact external abort callback on first settlement or teardown', async () => {
  541. const { ctx, parent } = await setup()
  542. const settledController = new AbortController()
  543. const settledAdd = vi.spyOn(settledController.signal, 'addEventListener')
  544. const settledRemove = vi.spyOn(settledController.signal, 'removeEventListener')
  545. const completed = ctx.workflows.start({ ...scripted('return 123'), parent, signal: settledController.signal })
  546. const settledAbort = settledAdd.mock.calls.find(([type]) => type === 'abort')?.[1]
  547. expect(typeof settledAbort).toBe('function')
  548. await expect(completed.result).resolves.toMatchObject({ value: 123, stopReason: 'completed' })
  549. expect(settledRemove).toHaveBeenCalledWith('abort', settledAbort)
  550. const cancelAfterSettle = vi.spyOn(completed, 'cancel')
  551. settledController.abort()
  552. expect(cancelAfterSettle).not.toHaveBeenCalled()
  553. cancelAfterSettle.mockRestore()
  554. await completed.dispose()
  555. const manual = await setup({ manual: true })
  556. const teardownController = new AbortController()
  557. const teardownAdd = vi.spyOn(teardownController.signal, 'addEventListener')
  558. const teardownRemove = vi.spyOn(teardownController.signal, 'removeEventListener')
  559. const tornDown = manual.ctx.workflows.start({
  560. ...scripted("return await agent('job')"),
  561. parent: manual.parent,
  562. signal: teardownController.signal,
  563. })
  564. await waitFor(() => { expect(manual.provider.runs).toHaveLength(1) })
  565. const teardownAbort = teardownAdd.mock.calls.find(([type]) => type === 'abort')?.[1]
  566. expect(typeof teardownAbort).toBe('function')
  567. const disposing = tornDown.dispose()
  568. expect(teardownRemove).toHaveBeenCalledWith('abort', teardownAbort)
  569. await disposing
  570. })
  571. it('a child-start racing the host cancel is refused: no child starts after cancellation', async () => {
  572. const { ctx, parent, provider } = await setup({ manual: true })
  573. // Cancel from INSIDE the log listener: the worker has already posted
  574. // its child-start (queued right behind the log message), so the host
  575. // processes it with cancelReason set — the refusal arm no real-world
  576. // timing can hit reliably. (The closure runs only after `handle` below
  577. // is initialized — the listener fires on the worker's first message.)
  578. ctx.on('workflow/log', () => { handle.cancel('cancelled from the log listener') })
  579. const handle = ctx.workflows.start({ ...scripted("log('mark')\nreturn await agent('late')"), parent })
  580. const result = await handle.result
  581. expect(result.stopReason).toBe('cancelled')
  582. expect(provider.runs.length).toBe(0)
  583. await handle.dispose()
  584. })
  585. it('post-cancel narration is suppressed host-side, and completion racing a cancel reports cancelled', async () => {
  586. const { ctx, parent } = await setup()
  587. const narration: string[] = []
  588. ctx.on('workflow/log', (_info, message) => { narration.push(message) })
  589. ctx.on('workflow/phase', (_info, title) => { narration.push(`phase:${title}`) })
  590. const handle = ctx.workflows.start({
  591. // The sync spin keeps the worker's loop busy so the cancel message
  592. // cannot be processed before the script settles `completed` — the
  593. // worker posts a completed result that must LOSE to the in-flight
  594. // host cancellation. The trailing narration exercises host-side
  595. // suppression: posted pre-cancel-processing worker-side, arriving
  596. // post-cancel host-side.
  597. ...scripted(`
  598. log('started')
  599. const end = Date.now() + 1000
  600. while (Date.now() < end) {}
  601. phase('late phase')
  602. log('late log')
  603. return 'done'
  604. `),
  605. parent,
  606. })
  607. await waitFor(() => { expect(narration).toContain('started') })
  608. handle.cancel('raced the completion')
  609. const result = await handle.result
  610. expect(result.stopReason).toBe('cancelled')
  611. expect(result.error).toContain('raced the completion')
  612. expect(narration).toEqual(['started'])
  613. await handle.dispose()
  614. }, 15_000)
  615. it('cancel() force-settles a script parked on a promise no hook owns, and TERMINATES its worker', async () => {
  616. const { ctx, parent } = await setup({ config: { provider: 'stub', disposeGraceMs: 50 } })
  617. const runEnds: WorkflowResultInfo[] = []
  618. ctx.on('workflow/end', (_info, result) => { runEnds.push(result) })
  619. const handle = ctx.workflows.start({
  620. ...scripted("await new Promise(() => {})\nreturn 'unreachable'"),
  621. parent,
  622. })
  623. handle.cancel('user aborted')
  624. const result = await handle.result
  625. expect(result.stopReason).toBe('cancelled')
  626. expect(result.error).toContain('user aborted')
  627. // The grace force-settle fires workflow/end exactly like an ordinary
  628. // settlement — a terminated script's death still reaches observers.
  629. expect(runEnds).toEqual([{ stopReason: 'cancelled', error: result.error, agentsStarted: 0 }])
  630. await handle.dispose()
  631. })
  632. it('dispose() on a stuck script returns within the grace instead of hanging (result settles cancelled)', async () => {
  633. const { ctx, parent } = await setup({ config: { provider: 'stub', disposeGraceMs: 50 } })
  634. const handle = ctx.workflows.start({
  635. ...scripted("await new Promise(() => {})\nreturn 'unreachable'"),
  636. parent,
  637. })
  638. const before = Date.now()
  639. await handle.dispose()
  640. expect(Date.now() - before).toBeLessThan(2000)
  641. const result = await handle.result
  642. expect(result.stopReason).toBe('cancelled')
  643. })
  644. it('dispose() is idempotent and settles cleanly after a completed run', async () => {
  645. const { ctx, parent } = await setup()
  646. const handle = ctx.workflows.start({ ...scripted('return 1'), parent })
  647. await handle.result
  648. await handle.dispose()
  649. await handle.dispose()
  650. })
  651. it('a settled run arms NO grace timer: disposing a completed run must not pin it for disposeGraceMs', async () => {
  652. // A distinctive grace so the spy can tell the cancel-path grace timer
  653. // apart from every other timeout in flight.
  654. const GRACE = 44_444
  655. const { ctx, parent } = await setup({ config: { provider: 'stub', disposeGraceMs: GRACE } })
  656. const handle = ctx.workflows.start({ ...scripted('return 1'), parent })
  657. await handle.result
  658. const spy = vi.spyOn(globalThis, 'setTimeout')
  659. try {
  660. await handle.dispose()
  661. // dispose()'s own bounded-wait sleep is the ONLY grace-sized timer
  662. // allowed here; before the settled guard, cancel() armed a second one
  663. // that nothing would ever clear (the run was already settled), keeping
  664. // the WorkerRun/Worker closure alive until the grace expired.
  665. const graceTimers = spy.mock.calls.filter(call => call[1] === GRACE)
  666. expect(graceTimers.length).toBe(1)
  667. } finally {
  668. spy.mockRestore()
  669. }
  670. })
  671. it('strays: children fired without await are aborted once the script settles, and dispose() waits for their disposal', async () => {
  672. const { ctx, parent, provider } = await setup({ manual: true, disposeDelayMs: 40 })
  673. const handle = ctx.workflows.start({
  674. ...scripted(`
  675. agent('stray')
  676. return 'done without awaiting'
  677. `),
  678. parent,
  679. })
  680. const result = await handle.result
  681. expect(result.stopReason).toBe('completed')
  682. await waitFor(() => { expect(provider.runs.length).toBe(1) })
  683. await handle.dispose()
  684. // Not a waitFor: by the time dispose() returns, the slow child disposal
  685. // must already be complete (host-side registry quiescence).
  686. expect(provider.runs[0]!.disposed).toBe(true)
  687. })
  688. it('result settlement reaps a registered stray even when the worker cannot relay disposal', async () => {
  689. const { ctx, parent, provider } = await setup({
  690. manual: true,
  691. config: { provider: 'stub', disposeGraceMs: 30_000 },
  692. })
  693. const handle = ctx.workflows.start({
  694. ...scripted("agent('stray')\nawait new Promise(() => {})"),
  695. parent,
  696. })
  697. await waitFor(() => { expect(provider.runs).toHaveLength(1) })
  698. // Claim the host result while the real worker remains wedged, so it can
  699. // send neither ChildDispose nor an exit. This leaves the accepted child
  700. // in the host registry when public disposal begins.
  701. const worker = (handle as unknown as { worker: Worker }).worker
  702. worker.emit('message', {
  703. type: WorkerToHostType.Result,
  704. result: { value: 'synthetic completion', stopReason: 'completed', agentsStarted: 1 },
  705. })
  706. await expect(handle.result).resolves.toMatchObject({ stopReason: 'completed' })
  707. await waitFor(() => { expect(provider.runs[0]!.disposed).toBe(true) }, 1000)
  708. const disposal = handle.dispose()
  709. await disposal
  710. expect(provider.runs[0]!.disposeCalls).toBe(1)
  711. await ctx.fiber.dispose()
  712. })
  713. it('the settle-reap fires the request signal too: a provider honoring ONLY the signal winds its stray down promptly', async () => {
  714. const ctx = new Context()
  715. await ctx.plugin(SubagentService)
  716. const aborted: string[] = []
  717. const provider: SubagentProvider = {
  718. name: 'signal-only',
  719. capabilities: { outputSchema: true, depthLimit: true, toolFilter: true, persona: false },
  720. inheritsParentContext: false,
  721. start: async (request) => {
  722. let settle!: (result: SubagentResult) => void
  723. const result = new Promise<SubagentResult>((resolve) => { settle = resolve })
  724. request.signal.addEventListener('abort', () => {
  725. aborted.push(String(request.signal.reason))
  726. settle({ output: [], stopReason: 'aborted' })
  727. }, { once: true })
  728. return {
  729. id: SessionId('signal-only-child'),
  730. localAgent: undefined,
  731. result,
  732. dispose: () => Promise.resolve(),
  733. }
  734. },
  735. }
  736. ctx.subagents.registerProvider(provider)
  737. await ctx.plugin(WorkerWorkflowEngine, { provider: 'signal-only', maxConcurrentAgents: 2 })
  738. const handle = ctx.workflows.start({
  739. ...scripted(`
  740. agent('stray, never awaited')
  741. return 'done'
  742. `),
  743. parent: fakeParent(),
  744. })
  745. const result = await handle.result
  746. expect(result.stopReason).toBe('completed')
  747. // BEFORE dispose(): the settlement itself must have aborted the signal —
  748. // without it this child would stay live until dispose's terminate. This
  749. // is a HOST-PROMPTNESS claim, not a cold-start race — a tight explicit
  750. // bound (unlike the file default) so a multi-second reap regression
  751. // cannot pass by outlasting the wait.
  752. await waitFor(() => { expect(aborted).toEqual(['workflow settled']) }, 1000)
  753. await handle.dispose()
  754. })
  755. it('the settle-reap aborts a pending provider start before workflow/end', async () => {
  756. const { ctx, parent, provider } = await setup({ manual: true, deferStart: true })
  757. const childLifecycle: string[] = []
  758. let cancellationAtWorkflowEnd: string | undefined
  759. ctx.on('workflow/agent-start', () => { childLifecycle.push('start') })
  760. ctx.on('workflow/agent-end', () => { childLifecycle.push('end') })
  761. ctx.on('workflow/end', () => {
  762. cancellationAtWorkflowEnd = provider.runs[0]?.cancelled
  763. })
  764. const handle = ctx.workflows.start({
  765. ...scripted(`
  766. agent('start-pending stray')
  767. return 'done'
  768. `),
  769. parent,
  770. })
  771. const result = await handle.result
  772. expect(result.stopReason).toBe('completed')
  773. expect(provider.runs).toHaveLength(1)
  774. expect(provider.runs[0]!.request.signal?.aborted).toBe(true)
  775. expect(provider.runs[0]!.request.signal?.reason).toBe('workflow settled')
  776. expect(provider.runs[0]!.cancelled).toBe('workflow settled')
  777. expect(cancellationAtWorkflowEnd).toBe('workflow settled')
  778. expect(childLifecycle).toEqual([])
  779. await handle.dispose()
  780. expect(provider.runs[0]!.disposeCalls).toBe(1)
  781. })
  782. it('a duplicate Result after the terminal claim cannot repeat cleanup or rewrite the outcome', async () => {
  783. let signalAborts = 0
  784. const { ctx, parent, provider } = await setup({
  785. manual: true,
  786. onChildAbortString: (_reason, index) => { if (index === 0) signalAborts += 1 },
  787. })
  788. const handle = ctx.workflows.start({
  789. ...scripted("agent('stray')\nawait new Promise(() => {})"),
  790. parent,
  791. })
  792. await waitFor(() => { expect(provider.runs).toHaveLength(1) })
  793. const worker = (handle as unknown as { worker: Worker }).worker
  794. worker.emit('message', {
  795. type: WorkerToHostType.Result,
  796. result: { value: 'first', stopReason: 'completed', agentsStarted: 1 },
  797. })
  798. worker.emit('message', {
  799. type: WorkerToHostType.Result,
  800. result: { value: 'late', stopReason: 'completed', agentsStarted: 1 },
  801. })
  802. await expect(handle.result).resolves.toMatchObject({ value: 'first', stopReason: 'completed' })
  803. expect(signalAborts).toBe(1)
  804. await handle.dispose()
  805. expect(signalAborts).toBe(1)
  806. await ctx.fiber.dispose()
  807. })
  808. it('a grace-terminated worker reaps its child on exit without waiting for consumer dispose()', async () => {
  809. const { ctx, parent, provider } = await setup({
  810. manual: true,
  811. config: { provider: 'stub', maxConcurrentAgents: 2, disposeGraceMs: 100 },
  812. })
  813. const handle = ctx.workflows.start({
  814. // Let child-start cross, then make the worker unable to process its
  815. // Cancel message. Grace settles the result and terminates the thread;
  816. // that exit must independently own the host registry's disposal pass.
  817. ...scripted(`
  818. agent('survives until exit reap')
  819. for (let i = 0; i < 20; i++) await null
  820. const end = Date.now() + 1500
  821. while (Date.now() < end) {}
  822. return 'unreachable'
  823. `),
  824. parent,
  825. })
  826. await waitFor(() => { expect(provider.runs).toHaveLength(1) })
  827. handle.cancel('force termination')
  828. const result = await handle.result
  829. expect(result.stopReason).toBe('cancelled')
  830. // Deliberately assert before handle.dispose(): host-owned worker exit,
  831. // not consumer courtesy, is responsible for this resource guarantee.
  832. await waitFor(() => { expect(provider.runs[0]!.disposed).toBe(true) }, 1000)
  833. expect(provider.runs[0]!.disposeCalls).toBe(1)
  834. await handle.dispose()
  835. await ctx.fiber.dispose()
  836. }, 15_000)
  837. it('dispose() on a wedged worker host-drives child disposal inside the grace: it returns with the children DISPOSED, not with their teardown still in flight', async () => {
  838. const { ctx, parent, provider } = await setup({
  839. manual: true,
  840. disposeDelayMs: 40,
  841. config: { provider: 'stub', maxConcurrentAgents: 8, disposeGraceMs: 400 },
  842. })
  843. const handle = ctx.workflows.start({
  844. // Same shape as the wedged-cancel test above: the child's start RPC
  845. // reaches the host, then the script seizes its worker's loop, so the
  846. // worker can relay NO dispose RPC — the host's own dispose() drive is
  847. // the only thing that can start (and finish) this child's disposal
  848. // before the grace runs out.
  849. ...scripted(`
  850. agent('wedged child')
  851. for (let i = 0; i < 20; i++) await null
  852. const end = Date.now() + 1500
  853. while (Date.now() < end) {}
  854. return 'raced'
  855. `),
  856. parent,
  857. })
  858. await waitFor(() => { expect(provider.runs.length).toBe(1) })
  859. const before = Date.now()
  860. await handle.dispose()
  861. // Bounded by the grace (plus the terminate), never by the 1.5s spin.
  862. expect(Date.now() - before).toBeLessThan(1200)
  863. // Not a waitFor: dispose() resolving IS the quiescence claim — the slow
  864. // child disposal must be complete, not merely started (before the
  865. // host-driven drive, disposal only STARTED at the post-terminate reap,
  866. // so dispose() returned with it still in flight).
  867. expect(provider.runs[0]!.disposed).toBe(true)
  868. const result = await handle.result
  869. expect(result.stopReason).toBe('cancelled')
  870. }, 15_000)
  871. it('a live child disposed by the dispose() drive is disposed ONCE, and the worker\'s late dispose RPC still gets its ack (the script settles, not the grace)', async () => {
  872. const { ctx, parent, provider } = await setup({ manual: true })
  873. const handle = ctx.workflows.start({
  874. ...scripted(`
  875. await agent('long child')
  876. return 'unreachable'
  877. `),
  878. parent,
  879. })
  880. await waitFor(() => { expect(provider.runs.length).toBe(1) })
  881. const handleDispose = handle.dispose()
  882. const result = await handle.result
  883. // The script itself settled (the wrapper's own dispose RPC found the
  884. // child already reaped host-side and was acked) — a missing ack would
  885. // wedge the wrapper's finally until the 5s default grace force-settle.
  886. expect(result.stopReason).toBe('cancelled')
  887. expect(result.error).toContain('workflow disposed')
  888. await handleDispose
  889. expect(provider.runs[0]!.disposed).toBe(true)
  890. // The memo: the host drive and the worker's RPC share one disposal.
  891. expect(provider.runs[0]!.disposeCalls).toBe(1)
  892. })
  893. it('the grace force-settle pairs every stranded start: a host-synthesized cancelled agent-end lands before workflow/end', async () => {
  894. const { ctx, parent, provider } = await setup({ manual: true, config: { provider: 'stub', maxConcurrentAgents: 8, disposeGraceMs: 300 } })
  895. const ends: { seq: number; outcome: string }[] = []
  896. const order: string[] = []
  897. ctx.on('workflow/agent-start', (_info, agent) => { order.push(`start:${agent.seq}`) })
  898. ctx.on('workflow/agent-end', (_info, agent) => {
  899. ends.push({ seq: agent.seq, outcome: agent.outcome })
  900. order.push(`end:${agent.seq}`)
  901. })
  902. ctx.on('workflow/end', () => { order.push('run-end') })
  903. const handle = ctx.workflows.start({
  904. // 'slow' starts and its agent-start crosses to observers (the awaited
  905. // 'fast' call keeps the worker loop turning), then the script seizes
  906. // the loop: the wedged worker can never author slow's agent-end —
  907. // only the host's ledger can close the pair.
  908. ...scripted(`
  909. const p = agent('slow')
  910. await agent('fast')
  911. const end = Date.now() + 1500
  912. while (Date.now() < end) {}
  913. return 'raced'
  914. `),
  915. parent,
  916. })
  917. await waitFor(() => { expect(order.filter(entry => entry.startsWith('start:')).length).toBe(2) })
  918. const fast = provider.runs.find(run => (run.request.prompt[0] as { text?: string }).text === 'fast')!
  919. fast.settle(text('fast done'))
  920. handle.cancel('stop now')
  921. const result = await handle.result
  922. expect(result.stopReason).toBe('cancelled')
  923. // fast's end is the worker's own report; slow's is host-synthesized at
  924. // the force-settle — exactly one end per started seq, no third event.
  925. expect(ends).toEqual([
  926. { seq: 2, outcome: 'completed' },
  927. { seq: 1, outcome: 'cancelled' },
  928. ])
  929. // Both ends reached observers BEFORE workflow/end: a progress consumer
  930. // can finalize its state at run-end without dangling agents.
  931. expect(order.indexOf('run-end')).toBe(order.length - 1)
  932. await handle.dispose()
  933. }, 15_000)
  934. it('graceful cancellation keeps pairing worker-authored: exactly one agent-end per start, nothing synthesized on top', async () => {
  935. const { ctx, parent, provider } = await setup({ manual: true })
  936. const ends: { seq: number; outcome: string }[] = []
  937. const order: string[] = []
  938. ctx.on('workflow/agent-end', (_info, agent) => {
  939. ends.push({ seq: agent.seq, outcome: agent.outcome })
  940. order.push(`end:${agent.seq}`)
  941. })
  942. ctx.on('workflow/end', () => { order.push('run-end') })
  943. const handle = ctx.workflows.start({
  944. ...scripted("await parallel([() => agent('a'), () => agent('b')])\nreturn 'unreachable'"),
  945. parent,
  946. })
  947. await waitFor(() => { expect(provider.runs.length).toBe(2) })
  948. handle.cancel('user stop')
  949. const result = await handle.result
  950. expect(result.stopReason).toBe('cancelled')
  951. // The live worker reported both pairs itself; the ledger must not add
  952. // a synthesized duplicate on any path that settles inside the grace.
  953. expect(ends.map(end => end.outcome)).toEqual(['cancelled', 'cancelled'])
  954. expect(new Set(ends.map(end => end.seq)).size).toBe(2)
  955. expect(order.indexOf('run-end')).toBe(order.length - 1)
  956. await handle.dispose()
  957. })
  958. })
  959. describe('worker death', () => {
  960. it('the first death signal closes admission to messages Node delivers before exit', async () => {
  961. const { ctx, parent, provider } = await setup({ manual: true })
  962. const phases: string[] = []
  963. ctx.on('workflow/phase', (_info, title) => { phases.push(title) })
  964. const handle = ctx.workflows.start({
  965. ...scripted('await new Promise(() => {})'),
  966. parent,
  967. })
  968. const worker = (handle as unknown as { worker: Worker }).worker
  969. // Node may physically emit error -> queued message -> exit. Reproduce
  970. // that ordering deterministically at the Worker event boundary: the
  971. // late protocol data must not create work, narrate, or rewrite error.
  972. worker.emit('error', new Error('synthetic error-before-message'))
  973. worker.emit('message', { type: WorkerToHostType.Phase, title: 'late phase' })
  974. worker.emit('message', {
  975. type: WorkerToHostType.ChildStart,
  976. callId: 999,
  977. request: { prompt: 'late child' },
  978. })
  979. worker.emit('message', {
  980. type: WorkerToHostType.Result,
  981. result: { value: 'late', stopReason: 'completed', agentsStarted: 1 },
  982. })
  983. const result = await handle.result
  984. expect(result.stopReason).toBe('error')
  985. expect(result.error).toContain('synthetic error-before-message')
  986. expect(provider.runs).toHaveLength(0)
  987. expect(phases).toEqual([])
  988. await handle.dispose()
  989. await ctx.fiber.dispose()
  990. })
  991. it('refuses and disposes a provider run that becomes ready after its real worker dies', async () => {
  992. const ctx = new Context()
  993. await ctx.plugin(SubagentService)
  994. const requested = Promise.withResolvers<SubagentStartRequest>()
  995. const ready = Promise.withResolvers<SubagentRun>()
  996. let disposeCalls = 0
  997. const warn = vi.spyOn(ctx.logger, 'warn').mockImplementation(() => ctx.logger)
  998. const provider: SubagentProvider = {
  999. name: 'late-ready',
  1000. capabilities: { outputSchema: true, depthLimit: true, toolFilter: true, persona: false },
  1001. inheritsParentContext: false,
  1002. start: (request) => {
  1003. requested.resolve(request)
  1004. // Model a backend whose independent startup boundary cannot be
  1005. // interrupted promptly. The host must still reject ownership if the
  1006. // worker dies before this promise transfers the ready run.
  1007. return ready.promise
  1008. },
  1009. }
  1010. ctx.subagents.registerProvider(provider)
  1011. await ctx.plugin(WorkerWorkflowEngine, { provider: 'late-ready', maxConcurrentAgents: 1 })
  1012. const lifecycle: string[] = []
  1013. ctx.on('workflow/agent-start', () => { lifecycle.push('start') })
  1014. ctx.on('workflow/agent-end', () => { lifecycle.push('end') })
  1015. const handle = ctx.workflows.start({
  1016. ...scripted("return await agent('pending startup')"),
  1017. parent: fakeParent(),
  1018. })
  1019. const request = await requested.promise
  1020. const worker = (handle as unknown as { worker: Worker }).worker
  1021. // Kill the actual Worker while provider startup is independently
  1022. // pending. Death closes admission and aborts the shared signal, but this
  1023. // deliberately uncooperative provider still fulfills afterward.
  1024. await worker.terminate()
  1025. const result = await handle.result
  1026. expect(result.stopReason).toBe('error')
  1027. expect(result.error).toContain('exit code')
  1028. expect(request.signal.aborted).toBe(true)
  1029. expect(request.signal.reason).toBe('workflow worker gone')
  1030. ready.resolve({
  1031. id: SessionId('late-ready-child'),
  1032. localAgent: undefined,
  1033. result: Promise.resolve({ output: [], stopReason: 'aborted' }),
  1034. dispose: () => {
  1035. disposeCalls += 1
  1036. return Promise.reject(new Error('late ready dispose failed'))
  1037. },
  1038. })
  1039. await waitFor(() => {
  1040. expect(disposeCalls).toBe(1)
  1041. expect(warn).toHaveBeenCalledWith(expect.stringContaining('refused child dispose failed: Error: late ready dispose failed'))
  1042. }, 1000)
  1043. expect(lifecycle).toEqual([])
  1044. await handle.dispose()
  1045. expect(disposeCalls).toBe(1)
  1046. await ctx.fiber.dispose()
  1047. })
  1048. it('a worker that exits before settling reports an error result and reaps its children', async () => {
  1049. const ctx = new Context()
  1050. await ctx.plugin(SubagentService)
  1051. // The child's dispose() REJECTS on top of the worker death: the reap
  1052. // must contain it (warn, not crash) while still emptying the registry.
  1053. const signalAborts: unknown[] = []
  1054. const provider: SubagentProvider = {
  1055. name: 'doomed',
  1056. capabilities: { outputSchema: true, depthLimit: true, toolFilter: true, persona: false },
  1057. inheritsParentContext: false,
  1058. start: async (request) => {
  1059. request.signal.addEventListener('abort', () => {
  1060. signalAborts.push(request.signal.reason)
  1061. // The death claim precedes the shared-signal fanout. This
  1062. // synchronous callback cannot turn death into cancellation.
  1063. handle.cancel('reentered from worker-death signal cleanup')
  1064. }, { once: true })
  1065. return {
  1066. id: SessionId('doomed-child'),
  1067. localAgent: undefined,
  1068. result: new Promise(() => { /* never settles; the reap is the teardown */ }),
  1069. dispose: () => Promise.reject(new Error('dispose exploded during reap')),
  1070. }
  1071. },
  1072. }
  1073. ctx.subagents.registerProvider(provider)
  1074. await ctx.plugin(WorkerWorkflowEngine, { provider: 'doomed', maxConcurrentAgents: 2 })
  1075. const runEnds: WorkflowResultInfo[] = []
  1076. ctx.on('workflow/end', (_info, result) => { runEnds.push(result) })
  1077. const handle = ctx.workflows.start({
  1078. // The stray child's start RPC reaches the host, then the script kills
  1079. // its own worker through the documented vm escape — the host must
  1080. // settle `error` with the exit diagnostics and wind the child down.
  1081. ...scripted(`
  1082. agent('doomed')
  1083. const proc = ${ESCAPE}
  1084. const st = globalThis.constructor.constructor('return setTimeout')()
  1085. await new Promise(resolve => st(resolve, 200))
  1086. proc.exit(7)
  1087. `),
  1088. parent: fakeParent(),
  1089. })
  1090. const result = await handle.result
  1091. expect(result.stopReason).toBe('error')
  1092. expect(result.error).toContain('exit code 7')
  1093. expect(result.agentsStarted).toBe(1)
  1094. // A worker death is a stop reason like any other: workflow/end fires
  1095. // with the error outcome — for a bus observer it is the only obituary.
  1096. expect(runEnds).toEqual([{ stopReason: 'error', error: result.error, agentsStarted: 1 }])
  1097. // Result already settled — this is the reap's promptness, not a
  1098. // cold-start race; tight explicit bound (see the helper's doc comment).
  1099. await waitFor(() => {
  1100. expect(signalAborts).toEqual(['workflow worker gone'])
  1101. }, 1000)
  1102. await Promise.resolve()
  1103. expect(result.stopReason).toBe('error')
  1104. await handle.dispose()
  1105. }, 15_000)
  1106. it('an uncaught exception inside the worker surfaces as an error result and reaps the in-flight child', async () => {
  1107. const { ctx, parent, provider } = await setup({ manual: true })
  1108. const handle = ctx.workflows.start({
  1109. ...scripted(`
  1110. agent('in flight when the worker dies')
  1111. const proc = ${ESCAPE}
  1112. const st = globalThis.constructor.constructor('return setTimeout')()
  1113. await new Promise(resolve => st(resolve, 200))
  1114. proc.nextTick(() => { throw new Error('worker blew up') })
  1115. await new Promise(() => {})
  1116. `),
  1117. parent,
  1118. })
  1119. const result = await handle.result
  1120. expect(result.stopReason).toBe('error')
  1121. expect(result.error).toContain('worker blew up')
  1122. // The reap wound the stray child down (cancel + a CLEAN dispose).
  1123. // Result already settled — this is the reap's promptness, not a
  1124. // cold-start race; tight explicit bound (see the helper's doc comment).
  1125. await waitFor(() => {
  1126. expect(provider.runs.length).toBe(1)
  1127. expect(provider.runs[0]!.disposed).toBe(true)
  1128. }, 1000)
  1129. await handle.dispose()
  1130. }, 15_000)
  1131. it('a worker death pairs every stranded start: the synthesized cancelled agent-end precedes the error workflow/end', async () => {
  1132. const { ctx, parent, provider } = await setup({ manual: true })
  1133. const ends: { seq: number; outcome: string }[] = []
  1134. const order: string[] = []
  1135. ctx.on('workflow/agent-start', (_info, agent) => { order.push(`start:${agent.seq}`) })
  1136. ctx.on('workflow/agent-end', (_info, agent) => {
  1137. ends.push({ seq: agent.seq, outcome: agent.outcome })
  1138. order.push(`end:${agent.seq}`)
  1139. })
  1140. ctx.on('workflow/end', () => { order.push('run-end') })
  1141. const handle = ctx.workflows.start({
  1142. // Same choreography as the force-settle pairing test, but the worker
  1143. // DIES (the documented vm escape) instead of being terminated: the
  1144. // exit path must close slow's pair from the ledger too. The escaped
  1145. // setTimeout lets the already-posted messages flush before the kill.
  1146. ...scripted(`
  1147. const p = agent('slow')
  1148. await agent('fast')
  1149. const proc = ${ESCAPE}
  1150. const st = globalThis.constructor.constructor('return setTimeout')()
  1151. await new Promise(resolve => st(resolve, 150))
  1152. proc.exit(7)
  1153. `),
  1154. parent,
  1155. })
  1156. await waitFor(() => { expect(order.filter(entry => entry.startsWith('start:')).length).toBe(2) })
  1157. const fast = provider.runs.find(run => (run.request.prompt[0] as { text?: string }).text === 'fast')!
  1158. fast.settle(text('fast done'))
  1159. const result = await handle.result
  1160. expect(result.stopReason).toBe('error')
  1161. expect(result.error).toContain('exit code 7')
  1162. expect(ends).toEqual([
  1163. { seq: 2, outcome: 'completed' },
  1164. { seq: 1, outcome: 'cancelled' },
  1165. ])
  1166. expect(order.indexOf('run-end')).toBe(order.length - 1)
  1167. await handle.dispose()
  1168. }, 15_000)
  1169. it('a dispose ack racing the worker death is dropped, not crashed (post after exit)', async () => {
  1170. // Slow child disposal: the ack resolves only AFTER the worker died, so
  1171. // it has nowhere to go and must be dropped silently (the workerGone
  1172. // guard in post()).
  1173. const { ctx, parent, provider } = await setup({ disposeDelayMs: 300 })
  1174. const handle = ctx.workflows.start({
  1175. // The STRAY child settles instantly, so its wrapper starts the slow
  1176. // host-side disposal concurrently while the script goes on to kill
  1177. // its own worker — the ack then resolves into a dead thread.
  1178. ...scripted(`
  1179. agent('stray, never awaited')
  1180. const proc = ${ESCAPE}
  1181. const st = globalThis.constructor.constructor('return setTimeout')()
  1182. await new Promise(resolve => st(resolve, 150))
  1183. proc.exit(5)
  1184. `),
  1185. parent,
  1186. })
  1187. const result = await handle.result
  1188. expect(result.stopReason).toBe('error')
  1189. expect(result.error).toContain('exit code 5')
  1190. // Result already settled — this is the reap's promptness (bounded
  1191. // above the mock's fixed 300ms dispose delay, not a cold-start race);
  1192. // tight explicit bound (see the helper's doc comment).
  1193. await waitFor(() => { expect(provider.runs[0]!.disposed).toBe(true) }, 1000)
  1194. await handle.dispose()
  1195. }, 15_000)
  1196. it('a worker death AFTER a cancel reports cancelled, not error', async () => {
  1197. const { ctx, parent } = await setup({ config: { provider: 'stub', disposeGraceMs: 60_000 } })
  1198. const handle = ctx.workflows.start({
  1199. ...scripted(`
  1200. const proc = ${ESCAPE}
  1201. const st = globalThis.constructor.constructor('return setTimeout')()
  1202. log('armed')
  1203. await new Promise(resolve => st(resolve, 400))
  1204. proc.exit(3)
  1205. `),
  1206. parent,
  1207. })
  1208. const logs: string[] = []
  1209. ctx.on('workflow/log', (_info, message) => { logs.push(message) })
  1210. await waitFor(() => { expect(logs).toContain('armed') })
  1211. handle.cancel('stop it')
  1212. // The grace is deliberately huge: only the worker's own death (exit 3,
  1213. // unreachable by the cancel — the script ignores hooks) settles this.
  1214. const result = await handle.result
  1215. expect(result.stopReason).toBe('cancelled')
  1216. expect(result.error).toContain('stop it')
  1217. await handle.dispose()
  1218. }, 15_000)
  1219. })
  1220. describe('service surface', () => {
  1221. it('run ids are unique and lifecycle meta is the run\'s borrowed immutable value', async () => {
  1222. const { ctx, parent } = await setup()
  1223. let eventMeta: WorkflowRunInfo | undefined
  1224. ctx.on('workflow/start', (info) => { eventMeta = info })
  1225. const first = ctx.workflows.start({ ...scripted('return 1'), parent })
  1226. const second = ctx.workflows.start({ ...scripted('return 2'), parent })
  1227. expect(first.id).not.toBe(second.id)
  1228. expect(eventMeta!.meta).toBe(second.meta)
  1229. expect(second.meta.name).toBe('test-flow')
  1230. await Promise.all([first.result, second.result])
  1231. await first.dispose()
  1232. await second.dispose()
  1233. })
  1234. it('unregisters ctx.workflows when the engine fiber is disposed (HMR safety)', async () => {
  1235. const ctx = new Context()
  1236. await ctx.plugin(SubagentService)
  1237. const fiber = await ctx.plugin(WorkerWorkflowEngine, {})
  1238. expect(ctx.get('workflows')).toBeDefined()
  1239. await fiber.dispose()
  1240. expect(ctx.get('workflows')).toBeUndefined()
  1241. })
  1242. it('keeps a holder-owned run usable when the engine unloads before its child starts', async () => {
  1243. const { ctx, parent, provider, engineFiber } = await setup({ reply: () => text('survived reload') })
  1244. let handle!: ReturnType<typeof ctx.workflows.start>
  1245. const holder = await ctx.plugin(Object.assign((inner: Context) => {
  1246. handle = inner.workflows.start({ ...scripted("return await agent('after reload')"), parent })
  1247. }, { inject: ['workflows'] }))
  1248. try {
  1249. // A real worker cannot deliver child-start in the synchronous start()
  1250. // slice. Unload the provider before that message arrives: the returned
  1251. // run belongs to `holder`, not to the engine fiber being reloaded.
  1252. expect(provider.runs).toHaveLength(0)
  1253. await engineFiber.dispose()
  1254. expect(ctx.get('workflows')).toBeUndefined()
  1255. await expect(handle.result).resolves.toEqual({
  1256. value: 'survived reload',
  1257. stopReason: 'completed',
  1258. agentsStarted: 1,
  1259. })
  1260. expect(provider.runs).toHaveLength(1)
  1261. } finally {
  1262. await handle.dispose()
  1263. await holder.dispose()
  1264. await ctx.fiber.dispose()
  1265. }
  1266. })
  1267. it('has the class-plugin export shape (default = the engine service class)', () => {
  1268. expect(workerEngineModule.default).toBe(WorkerWorkflowEngine)
  1269. expect('WorkerWorkflowEngine' in workerEngineModule).toBe(false)
  1270. const loader = Object.create(Loader.prototype) as Loader
  1271. const unwrapped: unknown = loader.unwrapExports(workerEngineModule)
  1272. expect(unwrapped).toBe(WorkerWorkflowEngine)
  1273. })
  1274. })
  1275. })