workflow-workerthread.spec.ts 60 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249
  1. import { describe, expect, it, vi } from 'vitest'
  2. import { fileURLToPath } from 'node:url'
  3. import { Context } from 'cordis'
  4. import Loader from '@cordisjs/plugin-loader'
  5. import { AgentId } from '@deepseek-ai/dsh-agent'
  6. import type { Agent } from '@deepseek-ai/dsh-agent'
  7. import SubagentService from '@deepseek-ai/dsh-subagent'
  8. import type { SubagentCapabilities, SubagentProvider, SubagentResult, SubagentRun, SubagentStartRequest } from '@deepseek-ai/dsh-subagent'
  9. import type { WorkflowMeta, WorkflowResult, WorkflowResultInfo, WorkflowRunInfo } from '@deepseek-ai/dsh-workflow'
  10. import * as workerEngineModule from '../src/index.ts'
  11. import WorkerWorkflowEngine, { HostToWorkerType, type Config } from '../src/index.ts'
  12. /** A minimal parent stand-in: the engine only threads it through to the provider. */
  13. function fakeParent(): Agent {
  14. return { id: AgentId('workflow-parent'), options: {} } as unknown as Agent
  15. }
  16. // Worker-thread startup is CPU-bound (a fresh thread compiles the runtime on
  17. // every start): on a contended CI runner it regularly blows past vitest's 5s
  18. // default test timeout, observed repeatedly on the coverage lane.
  19. vi.setConfig({ testTimeout: 30_000 })
  20. /**
  21. * `vi.waitFor` with a contention-proof default timeout: the 1s default
  22. * flaked repeatedly on the CI coverage lane, where worker-thread cold start
  23. * (CPU-bound — a fresh thread compiles the runtime) competes with three
  24. * sibling vitest workers for CPU. The 10s default is for exactly those
  25. * races — waiting for a worker to start, run its first script line, or
  26. * deliver an async child-registration message to the host. It is NOT for a
  27. * wait that asserts the HOST reacted PROMPTLY to something that already
  28. * happened (a settled result, an observed worker death): those keep an
  29. * explicit tight override below, or the generous default would silently
  30. * accept a multi-second regression in host-side reap latency as passing
  31. * (proven by injecting a 6s delay into one such reap and watching the
  32. * un-overridden version of this helper still pass in ~6s).
  33. * @param assertion - retried until it stops throwing or the timeout elapses.
  34. * @param timeout - override for a wait that must stay deliberately tight.
  35. * @returns resolves when the assertion passes.
  36. */
  37. function waitFor(assertion: () => void, timeout = 10_000): Promise<void> {
  38. return vi.waitFor(assertion, { timeout, interval: 50 })
  39. }
  40. /** The vm-context escape hatch, spelled once: real Worker tests use it to make the WORKER misbehave. */
  41. const ESCAPE = "globalThis.constructor.constructor('return process')()"
  42. /** One controllable child run: the test (or auto mode) settles it. */
  43. interface ControlledRun {
  44. request: SubagentStartRequest
  45. /** Fulfill the provider publication/readiness boundary. */
  46. publish(): void
  47. /** Reject the provider publication/readiness boundary. */
  48. rejectStart(error: unknown): void
  49. settle(result: SubagentResult): void
  50. rejectResult(error: unknown): void
  51. cancelled: string | undefined
  52. disposed: boolean
  53. disposeCalls: number
  54. }
  55. /**
  56. * A scripted in-test provider over the REAL SubagentService registry: `auto`
  57. * settles each run via the reply function on a microtask; `manual` piles runs
  58. * up in `runs` for the test to settle. A run aborts (settles `aborted`) when
  59. * the request signal fires, like the real in-process backends.
  60. */
  61. class StubProvider implements SubagentProvider {
  62. readonly capabilities: SubagentCapabilities = { outputSchema: true, depthLimit: true, toolFilter: true, persona: false }
  63. readonly inheritsParentContext = false
  64. readonly runs: ControlledRun[] = []
  65. constructor(
  66. readonly name: string,
  67. private readonly reply?: (request: SubagentStartRequest, index: number) => SubagentResult,
  68. private readonly disposeDelayMs = 0,
  69. private readonly deferStart = false,
  70. ) {}
  71. start(request: SubagentStartRequest): SubagentRun {
  72. const readiness = Promise.withResolvers<undefined>()
  73. const terminal = Promise.withResolvers<SubagentResult>()
  74. const controlled: ControlledRun = {
  75. request,
  76. publish: () => { readiness.resolve(undefined) },
  77. rejectStart: (error) => { readiness.reject(error) },
  78. settle: (result) => { terminal.resolve(result) },
  79. rejectResult: (error) => { terminal.reject(error) },
  80. cancelled: undefined,
  81. disposed: false,
  82. disposeCalls: 0,
  83. }
  84. this.runs.push(controlled)
  85. const index = this.runs.length - 1
  86. request.signal?.addEventListener('abort', () => { terminal.resolve({ output: [], stopReason: 'aborted' }) }, { once: true })
  87. if (!this.deferStart) readiness.resolve(undefined)
  88. if (this.reply) {
  89. const reply = this.reply
  90. queueMicrotask(() => { terminal.resolve(reply(request, index)) })
  91. }
  92. return {
  93. id: AgentId(`stub-child-${index}`),
  94. started: readiness.promise,
  95. result: terminal.promise,
  96. cancel: (reason?: string) => {
  97. controlled.cancelled = reason ?? 'cancelled'
  98. terminal.resolve({ output: [], stopReason: 'aborted' })
  99. },
  100. dispose: () => {
  101. controlled.disposeCalls += 1
  102. if (this.disposeDelayMs === 0) {
  103. controlled.disposed = true
  104. return Promise.resolve()
  105. }
  106. return new Promise<void>((resolve) => {
  107. setTimeout(() => {
  108. controlled.disposed = true
  109. resolve()
  110. }, this.disposeDelayMs)
  111. })
  112. },
  113. }
  114. }
  115. }
  116. /** Text-reply helper for auto providers. */
  117. function text(reply: string): SubagentResult {
  118. return { output: [{ type: 'text', text: reply }], stopReason: 'completed' }
  119. }
  120. interface SetupOptions {
  121. config?: Config
  122. reply?: (request: SubagentStartRequest, index: number) => SubagentResult
  123. manual?: boolean
  124. disposeDelayMs?: number
  125. deferStart?: boolean
  126. }
  127. async function setup(options?: SetupOptions) {
  128. const ctx = new Context()
  129. await ctx.plugin(SubagentService)
  130. const provider = new StubProvider(
  131. 'stub',
  132. options?.manual ? undefined : options?.reply ?? (() => text('stub reply')),
  133. options?.disposeDelayMs ?? 0,
  134. options?.deferStart ?? false,
  135. )
  136. ctx.subagents.registerProvider(provider)
  137. // A fixed concurrency ceiling: the auto-resolved default is machine-derived
  138. // (cores - 2, floored at 1), so tests that expect N children in flight
  139. // would wedge on small CI runners.
  140. await ctx.plugin(WorkerWorkflowEngine, { provider: 'stub', maxConcurrentAgents: 8, ...options?.config })
  141. return { ctx, provider, parent: fakeParent() }
  142. }
  143. /** The standard test meta plus a body, spread into a start request. */
  144. function scripted(body: string, metaExtra?: Partial<WorkflowMeta>): { script: string; meta: WorkflowMeta } {
  145. return { script: body, meta: { name: 'test-flow', description: 'a test workflow', ...metaExtra } }
  146. }
  147. /** Start + await one run, disposing on the way out. */
  148. async function run(ctx: Context, parent: Agent, source: { script: string; meta: WorkflowMeta }, args?: unknown): Promise<WorkflowResult> {
  149. const handle = ctx.workflows.start({ ...source, parent, ...args !== undefined ? { args } : {} })
  150. try {
  151. return await handle.result
  152. } finally {
  153. await handle.dispose()
  154. }
  155. }
  156. describe('dsh-workflow-workerthread', () => {
  157. describe('script execution over a real worker thread', () => {
  158. it('runs a script end-to-end: agent() text results, phases, log, args, return value, events', async () => {
  159. const { ctx, parent, provider } = await setup({ reply: (_request, index) => text(`answer-${index}`) })
  160. const events: [string, unknown[]][] = []
  161. for (const name of ['workflow/start', 'workflow/phase', 'workflow/log', 'workflow/agent-start', 'workflow/agent-end', 'workflow/end'] as const) {
  162. ctx.on(name, (...payload: unknown[]) => { events.push([name, payload]) })
  163. }
  164. const result = await run(ctx, parent, scripted(`
  165. phase('Scan')
  166. log('starting with ' + args.files.length + ' files')
  167. const answers = await pipeline(args.files, (prev, item) => agent('read ' + item))
  168. phase('Report')
  169. return { answers, count: args.files.length }
  170. `, { phases: [{ title: 'Scan' }, { title: 'Report' }] }), { files: ['a.ts', 'b.ts'] })
  171. expect(result.stopReason).toBe('completed')
  172. expect(result.agentsStarted).toBe(2)
  173. expect(result.value).toEqual({ answers: ['answer-0', 'answer-1'], count: 2 })
  174. expect(provider.runs.every(r => r.disposed)).toBe(true)
  175. const names = events.map(([name]) => name)
  176. expect(names[0]).toBe('workflow/start')
  177. expect(names).toContain('workflow/phase')
  178. expect(names).toContain('workflow/log')
  179. expect(names.at(-1)).toBe('workflow/end')
  180. const info = events[0]![1][0] as WorkflowRunInfo
  181. expect(info.meta.name).toBe('test-flow')
  182. const end = events.at(-1)![1][1] as Record<string, unknown>
  183. expect(end).toEqual({ stopReason: 'completed', agentsStarted: 2 })
  184. expect('value' in end).toBe(false)
  185. })
  186. it('agent({schema, model}) forwards outputSchema and agentOptions to the provider across the thread', async () => {
  187. const { ctx, parent, provider } = await setup({
  188. reply: () => ({ output: [], structured: { files: ['x.ts', 'y.ts'] }, stopReason: 'completed' }),
  189. })
  190. const result = await run(ctx, parent, scripted(`
  191. const found = await agent('list files', { model: 'deepseek-v4-pro', schema: { type: 'object', properties: { files: { type: 'array', items: { type: 'string' } } }, required: ['files'] } })
  192. return { first: found.files[0], count: found.files.length }
  193. `))
  194. expect(result.value).toEqual({ first: 'x.ts', count: 2 })
  195. expect(provider.runs[0]!.request.outputSchema).toEqual({
  196. type: 'object',
  197. properties: { files: { type: 'array', items: { type: 'string' } } },
  198. required: ['files'],
  199. })
  200. expect(provider.runs[0]!.request.agentOptions).toEqual({ model: 'deepseek-v4-pro' })
  201. expect(provider.runs[0]!.request.parent).toBeDefined()
  202. })
  203. it('a fatal hook error inside the worker kills the script and reports the error', async () => {
  204. const { ctx, parent } = await setup()
  205. const result = await run(ctx, parent, scripted("return await parallel([() => agent('x', { isolation: 'worktree' })])"))
  206. expect(result.stopReason).toBe('error')
  207. expect(result.error).toContain('"isolation" is deferred')
  208. })
  209. it('a provider start failure crosses back as a fatal AGENT_START error', async () => {
  210. const { ctx, parent } = await setup({ config: { provider: 'nonexistent' } })
  211. const result = await run(ctx, parent, scripted("return await pipeline([1], () => agent('p'))"))
  212. expect(result.stopReason).toBe('error')
  213. expect(result.error).toContain('agent() could not start a child')
  214. })
  215. it('waits for child readiness before announcing it and snapshots a result that settled early', async () => {
  216. const { ctx, parent, provider } = await setup({ manual: true, deferStart: true })
  217. const order: string[] = []
  218. ctx.on('workflow/agent-start', (_info, agent) => { order.push(`start:${agent.seq}`) })
  219. ctx.on('workflow/agent-end', (_info, agent) => { order.push(`end:${agent.outcome}`) })
  220. ctx.on('workflow/end', () => { order.push('run-end') })
  221. const handle = ctx.workflows.start({ ...scripted("return await agent('p')"), parent })
  222. await waitFor(() => { expect(provider.runs.length).toBe(1) })
  223. const early = text('accepted value')
  224. provider.runs[0]!.settle(early)
  225. // Let the host observe + snapshot result while readiness remains pending.
  226. await new Promise(resolve => setTimeout(resolve, 0))
  227. const earlyText = early.output[0] as { type: 'text'; text: string }
  228. earlyText.text = 'mutated after settlement'
  229. expect(order).toEqual([])
  230. provider.runs[0]!.publish()
  231. const result = await handle.result
  232. expect(result.value).toBe('accepted value')
  233. expect(order).toEqual(['start:1', 'end:completed', 'run-end'])
  234. await handle.dispose()
  235. expect(provider.runs[0]!.disposeCalls).toBe(1)
  236. })
  237. it('observes an early result rejection but sends ChildStarted before ChildFailed after readiness', async () => {
  238. const { ctx, parent, provider } = await setup({ manual: true, deferStart: true })
  239. const lifecycle: string[] = []
  240. ctx.on('workflow/agent-start', () => { lifecycle.push('start') })
  241. ctx.on('workflow/agent-end', (_info, agent) => { lifecycle.push(`end:${agent.outcome}`) })
  242. const handle = ctx.workflows.start({
  243. ...scripted("try { await agent('p'); return 'unreachable' } catch (e) { return { code: e.code, message: e.message } }"),
  244. parent,
  245. })
  246. const worker = (handle as unknown as { worker: { postMessage(message: unknown): void } }).worker
  247. const post = vi.spyOn(worker, 'postMessage')
  248. const childMessageTypes = (): HostToWorkerType[] => post.mock.calls
  249. .map(([message]) => (message as { type: HostToWorkerType }).type)
  250. .filter(type => type === HostToWorkerType.ChildStarted || type === HostToWorkerType.ChildFailed)
  251. await waitFor(() => { expect(provider.runs.length).toBe(1) })
  252. provider.runs[0]!.rejectResult(new Error('backend failed before publication'))
  253. await new Promise(resolve => setTimeout(resolve, 0))
  254. expect(childMessageTypes()).toEqual([])
  255. expect(lifecycle).toEqual([])
  256. provider.runs[0]!.publish()
  257. const result = await handle.result
  258. expect(result.value).toMatchObject({ code: 'AGENT_RESULT' })
  259. expect((result.value as { message: string }).message).toContain('backend failed before publication')
  260. expect(childMessageTypes()).toEqual([HostToWorkerType.ChildStarted, HostToWorkerType.ChildFailed])
  261. expect(lifecycle).toEqual(['start', 'end:failed'])
  262. post.mockRestore()
  263. await handle.dispose()
  264. })
  265. it('classifies readiness rejection as AGENT_START, drops an early result, and emits no false lifecycle pair', async () => {
  266. const { ctx, parent, provider } = await setup({ manual: true, deferStart: true })
  267. const lifecycle: string[] = []
  268. ctx.on('workflow/agent-start', () => { lifecycle.push('start') })
  269. ctx.on('workflow/agent-end', () => { lifecycle.push('end') })
  270. const handle = ctx.workflows.start({
  271. ...scripted("try { await agent('p'); return 'unreachable' } catch (e) { return { code: e.code, message: e.message } }"),
  272. parent,
  273. })
  274. await waitFor(() => { expect(provider.runs.length).toBe(1) })
  275. // ACP-style failure can settle result(error) before its session/publication
  276. // boundary rejects. Readiness must dominate that buffered child outcome.
  277. provider.runs[0]!.settle({ output: [], stopReason: 'error' })
  278. await new Promise(resolve => setTimeout(resolve, 0))
  279. provider.runs[0]!.rejectStart(new Error('publication rolled back'))
  280. const result = await handle.result
  281. expect(result.value).toMatchObject({ code: 'AGENT_START' })
  282. expect((result.value as { message: string }).message).toContain('publication rolled back')
  283. expect(lifecycle).toEqual([])
  284. await waitFor(() => {
  285. expect(provider.runs[0]!.disposed).toBe(true)
  286. expect(provider.runs[0]!.disposeCalls).toBe(1)
  287. })
  288. await handle.dispose()
  289. expect(provider.runs[0]!.disposeCalls).toBe(1)
  290. })
  291. it('cancels and disposes a readiness-pending child once without publishing workflow lifecycle', async () => {
  292. const { ctx, parent, provider } = await setup({ manual: true, deferStart: true, config: { disposeGraceMs: 500 } })
  293. const lifecycle: string[] = []
  294. ctx.on('workflow/agent-start', () => { lifecycle.push('start') })
  295. ctx.on('workflow/agent-end', () => { lifecycle.push('end') })
  296. const handle = ctx.workflows.start({ ...scripted("return await agent('pending')"), parent })
  297. await waitFor(() => { expect(provider.runs.length).toBe(1) })
  298. const disposal = handle.dispose()
  299. await waitFor(() => {
  300. expect(provider.runs[0]!.cancelled).toBe('workflow disposed')
  301. expect(provider.runs[0]!.disposed).toBe(true)
  302. })
  303. // Ensure the host-driven disposal removed the registry entry before the
  304. // late readiness rejection; its callback must not invoke dispose again.
  305. await new Promise(resolve => setTimeout(resolve, 0))
  306. provider.runs[0]!.rejectStart(new Error('cancelled before publication'))
  307. const result = await handle.result
  308. await disposal
  309. expect(result.stopReason).toBe('cancelled')
  310. expect(lifecycle).toEqual([])
  311. expect(provider.runs[0]!.disposeCalls).toBe(1)
  312. })
  313. it('a child result REJECTION crosses back as a fatal AGENT_RESULT error (a broken provider is not a failed child)', async () => {
  314. const ctx = new Context()
  315. await ctx.plugin(SubagentService)
  316. const provider: SubagentProvider = {
  317. name: 'rejecting',
  318. capabilities: { outputSchema: true, depthLimit: true, toolFilter: true, persona: false },
  319. inheritsParentContext: false,
  320. start: () => ({
  321. id: AgentId('reject-child'),
  322. started: Promise.resolve(),
  323. result: Promise.reject(new Error('backend exploded')),
  324. cancel: () => { /* nothing in flight */ },
  325. dispose: () => Promise.resolve(),
  326. }),
  327. }
  328. ctx.subagents.registerProvider(provider)
  329. await ctx.plugin(WorkerWorkflowEngine, { provider: 'rejecting', maxConcurrentAgents: 2 })
  330. const result = await run(ctx, fakeParent(), scripted(`
  331. try { await agent('p'); return 'unreachable' } catch (e) { return { name: e.name, code: e.code, fatal: e.fatal, message: e.message } }
  332. `))
  333. expect(result.value).toMatchObject({ name: 'WorkflowError', code: 'AGENT_RESULT', fatal: true })
  334. expect((result.value as { message: string }).message).toContain('backend exploded')
  335. })
  336. it('maps a non-JSON ready-child result to fatal AGENT_RESULT instead of wedging the bridge', async () => {
  337. const { ctx, parent } = await setup({
  338. reply: () => ({ output: [], structured: () => { /* deliberately outside lossless JSON */ }, stopReason: 'completed' }),
  339. })
  340. const result = await run(ctx, parent, scripted(`
  341. try { await agent('p'); return 'unreachable' } catch (e) { return { code: e.code, message: e.message } }
  342. `))
  343. expect(result.value).toMatchObject({ code: 'AGENT_RESULT' })
  344. expect((result.value as { message: string }).message).toContain('subagent result must be losslessly JSON-serializable')
  345. })
  346. it('contains a non-JSON result even if the injected subagent service violates its normalization contract', async () => {
  347. // SubagentService normally rejects this before the workflow sees it. Stub
  348. // the injected seam itself so the host's defensive worker-boundary guard
  349. // remains independently covered rather than becoming dead, untested code.
  350. const { ctx, parent } = await setup()
  351. const invalid = {
  352. output: [],
  353. structured: () => { /* deliberately outside lossless JSON */ },
  354. stopReason: 'completed',
  355. } as unknown as SubagentResult
  356. const start = vi.spyOn(ctx.subagents, 'start').mockReturnValue({
  357. id: AgentId('raw-invalid-child'),
  358. started: Promise.resolve(),
  359. result: Promise.resolve(invalid),
  360. cancel: () => { /* already settled */ },
  361. dispose: () => Promise.resolve(),
  362. })
  363. const result = await run(ctx, parent, scripted(`
  364. try { await agent('p'); return 'unreachable' } catch (e) { return { code: e.code, message: e.message } }
  365. `))
  366. expect(start).toHaveBeenCalledOnce()
  367. expect(result.value).toMatchObject({ code: 'AGENT_RESULT' })
  368. expect((result.value as { message: string }).message)
  369. .toContain('workflow child result could not cross the worker boundary')
  370. })
  371. it('reads each resolved child-result field once before crossing the worker boundary', async () => {
  372. let structuredReads = 0
  373. class DriftedStructured { readonly value = 'drifted' }
  374. const { ctx, parent } = await setup({
  375. reply: () => ({
  376. output: [],
  377. get structured() {
  378. structuredReads += 1
  379. return structuredReads === 1 ? { value: 'accepted' } : new DriftedStructured()
  380. },
  381. stopReason: 'completed',
  382. }),
  383. })
  384. const result = await run(ctx, parent, scripted(`
  385. const found = await agent('p', {
  386. schema: { type: 'object', properties: { value: { type: 'string' } }, required: ['value'] }
  387. })
  388. return found.value
  389. `))
  390. expect(result.value).toBe('accepted')
  391. expect(structuredReads).toBe(1)
  392. })
  393. it('a child whose dispose() throws synchronously cannot wedge the script (the host acks anyway)', async () => {
  394. const ctx = new Context()
  395. await ctx.plugin(SubagentService)
  396. const provider: SubagentProvider = {
  397. name: 'bad-dispose',
  398. capabilities: { outputSchema: true, depthLimit: true, toolFilter: true, persona: false },
  399. inheritsParentContext: false,
  400. start: () => ({
  401. id: AgentId('bad-dispose-child'),
  402. started: Promise.resolve(),
  403. result: Promise.resolve({ output: [{ type: 'text', text: 'fine' }], stopReason: 'completed' }),
  404. cancel: () => { /* settled already */ },
  405. dispose: () => { throw new Error('dispose exploded') },
  406. }),
  407. }
  408. ctx.subagents.registerProvider(provider)
  409. await ctx.plugin(WorkerWorkflowEngine, { provider: 'bad-dispose', maxConcurrentAgents: 2 })
  410. const result = await run(ctx, fakeParent(), scripted("return await agent('p')"))
  411. expect(result.stopReason).toBe('completed')
  412. expect(result.value).toBe('fine')
  413. })
  414. it('a child dispose() rejecting an UNRENDERABLE value still acks — the containment warn is total', async () => {
  415. const ctx = new Context()
  416. await ctx.plugin(SubagentService)
  417. const provider: SubagentProvider = {
  418. name: 'coercion-trap-dispose',
  419. capabilities: { outputSchema: true, depthLimit: true, toolFilter: true, persona: false },
  420. inheritsParentContext: false,
  421. start: () => ({
  422. id: AgentId('trap-child'),
  423. started: Promise.resolve(),
  424. result: Promise.resolve({ output: [{ type: 'text', text: 'fine' }], stopReason: 'completed' }),
  425. cancel: () => { /* settled already */ },
  426. // The rejection VALUE's own coercion throws: a warn built with bare
  427. // String(error) would itself throw, skipping the ChildDisposed ack
  428. // and wedging the script's finally until the grace/terminate path.
  429. // eslint-disable-next-line @typescript-eslint/prefer-promise-reject-errors -- the non-Error rejection IS the scenario under test
  430. dispose: () => Promise.reject({ toString: () => { throw new Error('coercion trap') } }),
  431. }),
  432. }
  433. ctx.subagents.registerProvider(provider)
  434. await ctx.plugin(WorkerWorkflowEngine, { provider: 'coercion-trap-dispose', maxConcurrentAgents: 2 })
  435. const result = await run(ctx, fakeParent(), scripted("return await agent('p')"))
  436. expect(result.stopReason).toBe('completed')
  437. expect(result.value).toBe('fine')
  438. })
  439. it('the worker spawns with an EMPTY environment: an escaped script finds no ambient credentials', async () => {
  440. const { ctx, parent } = await setup()
  441. // A canary in the HARNESS process's env: with an inherited environment
  442. // the escape below would read it back (exactly how DEEPSEEK_API_KEY
  443. // would leak); env: {} in the spawn options is what keeps it out.
  444. process.env.WORKFLOW_ENV_CANARY = 'leak me'
  445. try {
  446. const result = await run(ctx, parent, scripted(`
  447. const proc = ${ESCAPE}
  448. return { canary: proc.env.WORKFLOW_ENV_CANARY ?? null, keys: Object.keys(proc.env).length }
  449. `))
  450. expect(result.stopReason).toBe('completed')
  451. expect(result.value).toEqual({ canary: null, keys: 0 })
  452. } finally {
  453. delete process.env.WORKFLOW_ENV_CANARY
  454. }
  455. })
  456. it('the unbuilt worker forwards exactly TSX_TSCONFIG_PATH through the scrub: the paths-map pin survives, secrets do not', async () => {
  457. const { ctx, parent } = await setup()
  458. // The ACP snapshot harness runs the parent with its cwd OUTSIDE the
  459. // repo and pins the repo tsconfig through this variable; the worker
  460. // must inherit the pin (or its dsh-* imports silently resolve to
  461. // unbuilt lib/ bundles) while every other variable stays scrubbed.
  462. const tsconfig = fileURLToPath(new URL('../../../../tsconfig.json', import.meta.url))
  463. process.env.TSX_TSCONFIG_PATH = tsconfig
  464. process.env.WORKFLOW_ENV_CANARY = 'leak me'
  465. try {
  466. const result = await run(ctx, parent, scripted(`
  467. const proc = ${ESCAPE}
  468. return { keys: Object.keys(proc.env), tsconfig: proc.env.TSX_TSCONFIG_PATH }
  469. `))
  470. expect(result.stopReason).toBe('completed')
  471. expect(result.value).toEqual({ keys: ['TSX_TSCONFIG_PATH'], tsconfig })
  472. } finally {
  473. delete process.env.TSX_TSCONFIG_PATH
  474. delete process.env.WORKFLOW_ENV_CANARY
  475. }
  476. })
  477. })
  478. describe('lifecycle: parse errors, cancellation, termination, disposal', () => {
  479. it('start() throws synchronously for invalid meta data or an unparseable body (host-side pre-checks)', async () => {
  480. const { ctx, parent } = await setup()
  481. // Meta is DATA — shape violations reject loud, every one named.
  482. expect(() => ctx.workflows.start({ script: 'return 1', meta: { name: '', description: 'd' }, parent })).toThrow(/meta\.name must be a non-empty string/)
  483. expect(() => ctx.workflows.start({ script: 'return 1', meta: { name: 'x', description: 'd', extra: 1 } as unknown as WorkflowMeta, parent })).toThrow(/META_INVALID|not a recognized field/)
  484. expect(() => ctx.workflows.start({ ...scripted('return ((('), parent })).toThrow(/does not parse/)
  485. // The likeliest authoring slip — a Claude Code-style meta header in the
  486. // body — gets a pointed message, not a bare SyntaxError.
  487. expect(() => ctx.workflows.start({ ...scripted("export const meta = { name: 'x', description: 'd' }\nreturn 1"), parent })).toThrow(/meta rides the `meta` request field/)
  488. })
  489. it('cancel() aborts in-flight children (signal AND cancel RPC) and settles the run cancelled', async () => {
  490. const { ctx, parent, provider } = await setup({ manual: true })
  491. const ends: unknown[] = []
  492. ctx.on('workflow/agent-end', (_info, agent) => { ends.push(agent) })
  493. const runEnds: WorkflowResultInfo[] = []
  494. ctx.on('workflow/end', (_info, result) => { runEnds.push(result) })
  495. const handle = ctx.workflows.start({ ...scripted("return await agent('long job')"), parent })
  496. await waitFor(() => { expect(provider.runs.length).toBe(1) })
  497. handle.cancel('user stopped it')
  498. const result = await handle.result
  499. expect(result.stopReason).toBe('cancelled')
  500. expect(result.error).toContain('user stopped it')
  501. await handle.dispose()
  502. expect(provider.runs[0]!.disposed).toBe(true)
  503. expect(ends).toEqual([expect.objectContaining({ seq: 1, outcome: 'cancelled' })])
  504. // workflow/end is an observer's only death signal: it fires for a
  505. // cancelled run too, mirroring the settled outcome data.
  506. expect(runEnds).toEqual([{ stopReason: 'cancelled', error: result.error, agentsStarted: result.agentsStarted }])
  507. })
  508. it('an already-aborted request signal cancels before the body ever runs (the go handshake holds it)', async () => {
  509. const { ctx, parent, provider } = await setup()
  510. const controller = new AbortController()
  511. controller.abort()
  512. const logs: string[] = []
  513. ctx.on('workflow/log', (_info, message) => { logs.push(message) })
  514. const handle = ctx.workflows.start({ ...scripted("log('ran')\nreturn 123"), parent, signal: controller.signal })
  515. const result = await handle.result
  516. expect(result.stopReason).toBe('cancelled')
  517. expect(result.value).toBeNull()
  518. expect(logs).toEqual([])
  519. expect(provider.runs.length).toBe(0)
  520. await handle.dispose()
  521. })
  522. it('cancel() right after start() cancels before the body runs; the signal aborting mid-run cancels like cancel()', async () => {
  523. const { ctx, parent, provider } = await setup({ manual: true })
  524. const first = ctx.workflows.start({ ...scripted("return await agent('never')"), parent })
  525. // No-reason cancel: the canonical default reason must ride the result.
  526. first.cancel()
  527. const firstResult = await first.result
  528. expect(firstResult.stopReason).toBe('cancelled')
  529. expect(firstResult.error).toContain('workflow cancelled')
  530. expect(provider.runs.length).toBe(0)
  531. await first.dispose()
  532. const controller = new AbortController()
  533. const second = ctx.workflows.start({ ...scripted("return await agent('job')"), parent, signal: controller.signal })
  534. await waitFor(() => { expect(provider.runs.length).toBe(1) })
  535. controller.abort()
  536. expect((await second.result).stopReason).toBe('cancelled')
  537. await second.dispose()
  538. })
  539. it('removes the exact external abort callback on first settlement or teardown', async () => {
  540. const { ctx, parent } = await setup()
  541. const settledController = new AbortController()
  542. const settledAdd = vi.spyOn(settledController.signal, 'addEventListener')
  543. const settledRemove = vi.spyOn(settledController.signal, 'removeEventListener')
  544. const completed = ctx.workflows.start({ ...scripted('return 123'), parent, signal: settledController.signal })
  545. const settledAbort = settledAdd.mock.calls.find(([type]) => type === 'abort')?.[1]
  546. expect(typeof settledAbort).toBe('function')
  547. await expect(completed.result).resolves.toMatchObject({ value: 123, stopReason: 'completed' })
  548. expect(settledRemove).toHaveBeenCalledWith('abort', settledAbort)
  549. const cancelAfterSettle = vi.spyOn(completed, 'cancel')
  550. settledController.abort()
  551. expect(cancelAfterSettle).not.toHaveBeenCalled()
  552. cancelAfterSettle.mockRestore()
  553. await completed.dispose()
  554. const manual = await setup({ manual: true })
  555. const teardownController = new AbortController()
  556. const teardownAdd = vi.spyOn(teardownController.signal, 'addEventListener')
  557. const teardownRemove = vi.spyOn(teardownController.signal, 'removeEventListener')
  558. const tornDown = manual.ctx.workflows.start({
  559. ...scripted("return await agent('job')"),
  560. parent: manual.parent,
  561. signal: teardownController.signal,
  562. })
  563. await waitFor(() => { expect(manual.provider.runs).toHaveLength(1) })
  564. const teardownAbort = teardownAdd.mock.calls.find(([type]) => type === 'abort')?.[1]
  565. expect(typeof teardownAbort).toBe('function')
  566. const disposing = tornDown.dispose()
  567. expect(teardownRemove).toHaveBeenCalledWith('abort', teardownAbort)
  568. await disposing
  569. })
  570. it('a child-start racing the host cancel is refused: no child starts after cancellation', async () => {
  571. const { ctx, parent, provider } = await setup({ manual: true })
  572. // Cancel from INSIDE the log listener: the worker has already posted
  573. // its child-start (queued right behind the log message), so the host
  574. // processes it with cancelReason set — the refusal arm no real-world
  575. // timing can hit reliably. (The closure runs only after `handle` below
  576. // is initialized — the listener fires on the worker's first message.)
  577. ctx.on('workflow/log', () => { handle.cancel('cancelled from the log listener') })
  578. const handle = ctx.workflows.start({ ...scripted("log('mark')\nreturn await agent('late')"), parent })
  579. const result = await handle.result
  580. expect(result.stopReason).toBe('cancelled')
  581. expect(provider.runs.length).toBe(0)
  582. await handle.dispose()
  583. })
  584. it('post-cancel narration is suppressed host-side, and completion racing a cancel reports cancelled', async () => {
  585. const { ctx, parent } = await setup()
  586. const narration: string[] = []
  587. ctx.on('workflow/log', (_info, message) => { narration.push(message) })
  588. ctx.on('workflow/phase', (_info, title) => { narration.push(`phase:${title}`) })
  589. const handle = ctx.workflows.start({
  590. // The sync spin keeps the worker's loop busy so the cancel message
  591. // cannot be processed before the script settles `completed` — the
  592. // worker posts a completed result that must LOSE to the in-flight
  593. // host cancellation. The trailing narration exercises host-side
  594. // suppression: posted pre-cancel-processing worker-side, arriving
  595. // post-cancel host-side.
  596. ...scripted(`
  597. log('started')
  598. const end = Date.now() + 1000
  599. while (Date.now() < end) {}
  600. phase('late phase')
  601. log('late log')
  602. return 'done'
  603. `),
  604. parent,
  605. })
  606. await waitFor(() => { expect(narration).toContain('started') })
  607. handle.cancel('raced the completion')
  608. const result = await handle.result
  609. expect(result.stopReason).toBe('cancelled')
  610. expect(result.error).toContain('raced the completion')
  611. expect(narration).toEqual(['started'])
  612. await handle.dispose()
  613. }, 15_000)
  614. it('cancel() force-settles a script parked on a promise no hook owns, and TERMINATES its worker', async () => {
  615. const { ctx, parent } = await setup({ config: { provider: 'stub', disposeGraceMs: 50 } })
  616. const runEnds: WorkflowResultInfo[] = []
  617. ctx.on('workflow/end', (_info, result) => { runEnds.push(result) })
  618. const handle = ctx.workflows.start({
  619. ...scripted("await new Promise(() => {})\nreturn 'unreachable'"),
  620. parent,
  621. })
  622. handle.cancel('user aborted')
  623. const result = await handle.result
  624. expect(result.stopReason).toBe('cancelled')
  625. expect(result.error).toContain('user aborted')
  626. // The grace force-settle fires workflow/end exactly like an ordinary
  627. // settlement — a terminated script's death still reaches observers.
  628. expect(runEnds).toEqual([{ stopReason: 'cancelled', error: result.error, agentsStarted: 0 }])
  629. await handle.dispose()
  630. })
  631. it('dispose() on a stuck script returns within the grace instead of hanging (result settles cancelled)', async () => {
  632. const { ctx, parent } = await setup({ config: { provider: 'stub', disposeGraceMs: 50 } })
  633. const handle = ctx.workflows.start({
  634. ...scripted("await new Promise(() => {})\nreturn 'unreachable'"),
  635. parent,
  636. })
  637. const before = Date.now()
  638. await handle.dispose()
  639. expect(Date.now() - before).toBeLessThan(2000)
  640. const result = await handle.result
  641. expect(result.stopReason).toBe('cancelled')
  642. })
  643. it('dispose() is idempotent and settles cleanly after a completed run', async () => {
  644. const { ctx, parent } = await setup()
  645. const handle = ctx.workflows.start({ ...scripted('return 1'), parent })
  646. await handle.result
  647. await handle.dispose()
  648. await handle.dispose()
  649. })
  650. it('a settled run arms NO grace timer: disposing a completed run must not pin it for disposeGraceMs', async () => {
  651. // A distinctive grace so the spy can tell the cancel-path grace timer
  652. // apart from every other timeout in flight.
  653. const GRACE = 44_444
  654. const { ctx, parent } = await setup({ config: { provider: 'stub', disposeGraceMs: GRACE } })
  655. const handle = ctx.workflows.start({ ...scripted('return 1'), parent })
  656. await handle.result
  657. const spy = vi.spyOn(globalThis, 'setTimeout')
  658. try {
  659. await handle.dispose()
  660. // dispose()'s own bounded-wait sleep is the ONLY grace-sized timer
  661. // allowed here; before the settled guard, cancel() armed a second one
  662. // that nothing would ever clear (the run was already settled), keeping
  663. // the WorkerRun/Worker closure alive until the grace expired.
  664. const graceTimers = spy.mock.calls.filter(call => call[1] === GRACE)
  665. expect(graceTimers.length).toBe(1)
  666. } finally {
  667. spy.mockRestore()
  668. }
  669. })
  670. it('strays: children fired without await are aborted once the script settles, and dispose() waits for their disposal', async () => {
  671. const { ctx, parent, provider } = await setup({ manual: true, disposeDelayMs: 40 })
  672. const handle = ctx.workflows.start({
  673. ...scripted(`
  674. agent('stray')
  675. return 'done without awaiting'
  676. `),
  677. parent,
  678. })
  679. const result = await handle.result
  680. expect(result.stopReason).toBe('completed')
  681. await waitFor(() => { expect(provider.runs.length).toBe(1) })
  682. await handle.dispose()
  683. // Not a waitFor: by the time dispose() returns, the slow child disposal
  684. // must already be complete (host-side registry quiescence).
  685. expect(provider.runs[0]!.disposed).toBe(true)
  686. })
  687. it('the settle-reap fires the request signal too: a provider honoring ONLY the signal winds its stray down promptly', async () => {
  688. const ctx = new Context()
  689. await ctx.plugin(SubagentService)
  690. const aborted: string[] = []
  691. const provider: SubagentProvider = {
  692. name: 'signal-only',
  693. capabilities: { outputSchema: true, depthLimit: true, toolFilter: true, persona: false },
  694. inheritsParentContext: false,
  695. start: (request) => {
  696. let settle!: (result: SubagentResult) => void
  697. const result = new Promise<SubagentResult>((resolve) => { settle = resolve })
  698. request.signal?.addEventListener('abort', () => {
  699. aborted.push(String(request.signal?.reason))
  700. settle({ output: [], stopReason: 'aborted' })
  701. }, { once: true })
  702. return {
  703. id: AgentId('signal-only-child'),
  704. started: Promise.resolve(),
  705. result,
  706. // The seam leaves a provider free to honor EITHER cancel channel;
  707. // this one deliberately ignores run.cancel() — only the request
  708. // signal can wind it down.
  709. cancel: () => { /* signal-only by design */ },
  710. dispose: () => Promise.resolve(),
  711. }
  712. },
  713. }
  714. ctx.subagents.registerProvider(provider)
  715. await ctx.plugin(WorkerWorkflowEngine, { provider: 'signal-only', maxConcurrentAgents: 2 })
  716. const handle = ctx.workflows.start({
  717. ...scripted(`
  718. agent('stray, never awaited')
  719. return 'done'
  720. `),
  721. parent: fakeParent(),
  722. })
  723. const result = await handle.result
  724. expect(result.stopReason).toBe('completed')
  725. // BEFORE dispose(): the settlement itself must have aborted the signal —
  726. // without it this child would stay live until dispose's terminate. This
  727. // is a HOST-PROMPTNESS claim, not a cold-start race — a tight explicit
  728. // bound (unlike the file default) so a multi-second reap regression
  729. // cannot pass by outlasting the wait.
  730. await waitFor(() => { expect(aborted).toEqual(['workflow settled']) }, 1000)
  731. await handle.dispose()
  732. })
  733. it('the settle-reap explicitly cancels a readiness-pending stray before workflow/end', async () => {
  734. const { ctx, parent, provider } = await setup({ manual: true, deferStart: true })
  735. const childLifecycle: string[] = []
  736. let cancellationAtWorkflowEnd: string | undefined
  737. ctx.on('workflow/agent-start', () => { childLifecycle.push('start') })
  738. ctx.on('workflow/agent-end', () => { childLifecycle.push('end') })
  739. ctx.on('workflow/end', () => {
  740. cancellationAtWorkflowEnd = provider.runs[0]?.cancelled
  741. })
  742. const handle = ctx.workflows.start({
  743. ...scripted(`
  744. agent('readiness-pending stray')
  745. return 'done'
  746. `),
  747. parent,
  748. })
  749. const result = await handle.result
  750. expect(result.stopReason).toBe('completed')
  751. expect(provider.runs).toHaveLength(1)
  752. expect(provider.runs[0]!.request.signal?.aborted).toBe(true)
  753. expect(provider.runs[0]!.request.signal?.reason).toBe('workflow settled')
  754. expect(provider.runs[0]!.cancelled).toBe('workflow settled')
  755. expect(cancellationAtWorkflowEnd).toBe('workflow settled')
  756. expect(childLifecycle).toEqual([])
  757. await handle.dispose()
  758. expect(provider.runs[0]!.disposeCalls).toBe(1)
  759. })
  760. it('contains a throwing child cancel and still settles after cancelling peer strays', async () => {
  761. const ctx = new Context()
  762. await ctx.plugin(SubagentService)
  763. let starts = 0
  764. const cancelled: string[] = []
  765. const warnings: string[] = []
  766. ctx.logger.warn = ((message: unknown) => { warnings.push(String(message)) }) as typeof ctx.logger.warn
  767. const provider: SubagentProvider = {
  768. name: 'throwing-cancel',
  769. capabilities: { outputSchema: true, depthLimit: true, toolFilter: true, persona: false },
  770. inheritsParentContext: false,
  771. start: () => {
  772. const index = starts++
  773. return {
  774. id: AgentId(`throwing-cancel-${index}`),
  775. started: new Promise(() => { /* readiness stays pending */ }),
  776. result: new Promise(() => { /* cancellation callback owns settlement */ }),
  777. cancel: (reason?: string) => {
  778. if (index === 0) throw new Error('cancel callback broke')
  779. cancelled.push(`${index}:${reason ?? 'cancelled'}`)
  780. },
  781. dispose: () => Promise.resolve(),
  782. }
  783. },
  784. }
  785. ctx.subagents.registerProvider(provider)
  786. await ctx.plugin(WorkerWorkflowEngine, { provider: 'throwing-cancel', maxConcurrentAgents: 2 })
  787. const handle = ctx.workflows.start({
  788. ...scripted(`
  789. agent('first stray')
  790. agent('second stray')
  791. return 'done'
  792. `),
  793. parent: fakeParent(),
  794. })
  795. const result = await handle.result
  796. expect(result.stopReason).toBe('completed')
  797. expect(starts).toBe(2)
  798. expect(cancelled).toContain('1:workflow settled')
  799. expect(warnings.some(message => message.includes('cancel callback broke'))).toBe(true)
  800. await handle.dispose()
  801. })
  802. it("cancel() drives each child's explicit cancel() host-side: a wedged worker cannot delay it", async () => {
  803. const ctx = new Context()
  804. await ctx.plugin(SubagentService)
  805. let starts = 0
  806. const cancelled: string[] = []
  807. const provider: SubagentProvider = {
  808. name: 'cancel-only',
  809. capabilities: { outputSchema: true, depthLimit: true, toolFilter: true, persona: false },
  810. inheritsParentContext: false,
  811. start: () => {
  812. starts += 1
  813. return {
  814. id: AgentId('cancel-only-child'),
  815. started: Promise.resolve(),
  816. result: new Promise(() => { /* only cancel() ends this child */ }),
  817. // Deliberately ignores the request signal — the seam leaves a
  818. // provider free to honor ONLY the explicit cancel() channel.
  819. cancel: (reason?: string) => { cancelled.push(reason ?? 'cancelled') },
  820. dispose: () => Promise.resolve(),
  821. }
  822. },
  823. }
  824. ctx.subagents.registerProvider(provider)
  825. // A deliberately huge grace: if only the grace/terminate reap could
  826. // reach this child, the assertion below would time out first.
  827. await ctx.plugin(WorkerWorkflowEngine, { provider: 'cancel-only', maxConcurrentAgents: 2, disposeGraceMs: 30_000 })
  828. const handle = ctx.workflows.start({
  829. // The stray child's start RPC reaches the host, then the script wedges
  830. // its own worker in a synchronous spin: the worker cannot process the
  831. // Cancel message, so it can relay NO ChildCancel RPC — only the host's
  832. // own children loop can deliver the explicit cancel in time. The
  833. // microtask yields let the agent() continuation POST its child-start
  834. // before the spin seizes the worker's loop (the posted message needs
  835. // no further worker-loop turns to reach the host).
  836. ...scripted(`
  837. agent('wedged child')
  838. for (let i = 0; i < 20; i++) await null
  839. const end = Date.now() + 1500
  840. while (Date.now() < end) {}
  841. return 'raced'
  842. `),
  843. parent: fakeParent(),
  844. })
  845. await waitFor(() => { expect(starts).toBe(1) })
  846. handle.cancel('stop now')
  847. await waitFor(() => { expect(cancelled).toEqual(['stop now']) }, 800)
  848. // The wedged worker's own completion loses to the in-flight cancel.
  849. const result = await handle.result
  850. expect(result.stopReason).toBe('cancelled')
  851. await handle.dispose()
  852. }, 15_000)
  853. it('dispose() on a wedged worker host-drives child disposal inside the grace: it returns with the children DISPOSED, not with their teardown still in flight', async () => {
  854. const { ctx, parent, provider } = await setup({
  855. manual: true,
  856. disposeDelayMs: 40,
  857. config: { provider: 'stub', maxConcurrentAgents: 8, disposeGraceMs: 400 },
  858. })
  859. const handle = ctx.workflows.start({
  860. // Same shape as the wedged-cancel test above: the child's start RPC
  861. // reaches the host, then the script seizes its worker's loop, so the
  862. // worker can relay NO dispose RPC — the host's own dispose() drive is
  863. // the only thing that can start (and finish) this child's disposal
  864. // before the grace runs out.
  865. ...scripted(`
  866. agent('wedged child')
  867. for (let i = 0; i < 20; i++) await null
  868. const end = Date.now() + 1500
  869. while (Date.now() < end) {}
  870. return 'raced'
  871. `),
  872. parent,
  873. })
  874. await waitFor(() => { expect(provider.runs.length).toBe(1) })
  875. const before = Date.now()
  876. await handle.dispose()
  877. // Bounded by the grace (plus the terminate), never by the 1.5s spin.
  878. expect(Date.now() - before).toBeLessThan(1200)
  879. // Not a waitFor: dispose() resolving IS the quiescence claim — the slow
  880. // child disposal must be complete, not merely started (before the
  881. // host-driven drive, disposal only STARTED at the post-terminate reap,
  882. // so dispose() returned with it still in flight).
  883. expect(provider.runs[0]!.disposed).toBe(true)
  884. const result = await handle.result
  885. expect(result.stopReason).toBe('cancelled')
  886. }, 15_000)
  887. it('a live child disposed by the dispose() drive is disposed ONCE, and the worker\'s late dispose RPC still gets its ack (the script settles, not the grace)', async () => {
  888. const { ctx, parent, provider } = await setup({ manual: true })
  889. const handle = ctx.workflows.start({
  890. ...scripted(`
  891. await agent('long child')
  892. return 'unreachable'
  893. `),
  894. parent,
  895. })
  896. await waitFor(() => { expect(provider.runs.length).toBe(1) })
  897. const handleDispose = handle.dispose()
  898. const result = await handle.result
  899. // The script itself settled (the wrapper's own dispose RPC found the
  900. // child already reaped host-side and was acked) — a missing ack would
  901. // wedge the wrapper's finally until the 5s default grace force-settle.
  902. expect(result.stopReason).toBe('cancelled')
  903. expect(result.error).toContain('workflow disposed')
  904. await handleDispose
  905. expect(provider.runs[0]!.disposed).toBe(true)
  906. // The memo: the host drive and the worker's RPC share one disposal.
  907. expect(provider.runs[0]!.disposeCalls).toBe(1)
  908. })
  909. it('the grace force-settle pairs every stranded start: a host-synthesized cancelled agent-end lands before workflow/end', async () => {
  910. const { ctx, parent, provider } = await setup({ manual: true, config: { provider: 'stub', maxConcurrentAgents: 8, disposeGraceMs: 300 } })
  911. const ends: { seq: number; outcome: string }[] = []
  912. const order: string[] = []
  913. ctx.on('workflow/agent-start', (_info, agent) => { order.push(`start:${agent.seq}`) })
  914. ctx.on('workflow/agent-end', (_info, agent) => {
  915. ends.push({ seq: agent.seq, outcome: agent.outcome })
  916. order.push(`end:${agent.seq}`)
  917. })
  918. ctx.on('workflow/end', () => { order.push('run-end') })
  919. const handle = ctx.workflows.start({
  920. // 'slow' starts and its agent-start crosses to observers (the awaited
  921. // 'fast' call keeps the worker loop turning), then the script seizes
  922. // the loop: the wedged worker can never author slow's agent-end —
  923. // only the host's ledger can close the pair.
  924. ...scripted(`
  925. const p = agent('slow')
  926. await agent('fast')
  927. const end = Date.now() + 1500
  928. while (Date.now() < end) {}
  929. return 'raced'
  930. `),
  931. parent,
  932. })
  933. await waitFor(() => { expect(order.filter(entry => entry.startsWith('start:')).length).toBe(2) })
  934. const fast = provider.runs.find(run => (run.request.prompt[0] as { text?: string }).text === 'fast')!
  935. fast.settle(text('fast done'))
  936. handle.cancel('stop now')
  937. const result = await handle.result
  938. expect(result.stopReason).toBe('cancelled')
  939. // fast's end is the worker's own report; slow's is host-synthesized at
  940. // the force-settle — exactly one end per started seq, no third event.
  941. expect(ends).toEqual([
  942. { seq: 2, outcome: 'completed' },
  943. { seq: 1, outcome: 'cancelled' },
  944. ])
  945. // Both ends reached observers BEFORE workflow/end: a progress consumer
  946. // can finalize its state at run-end without dangling agents.
  947. expect(order.indexOf('run-end')).toBe(order.length - 1)
  948. await handle.dispose()
  949. }, 15_000)
  950. it('graceful cancellation keeps pairing worker-authored: exactly one agent-end per start, nothing synthesized on top', async () => {
  951. const { ctx, parent, provider } = await setup({ manual: true })
  952. const ends: { seq: number; outcome: string }[] = []
  953. const order: string[] = []
  954. ctx.on('workflow/agent-end', (_info, agent) => {
  955. ends.push({ seq: agent.seq, outcome: agent.outcome })
  956. order.push(`end:${agent.seq}`)
  957. })
  958. ctx.on('workflow/end', () => { order.push('run-end') })
  959. const handle = ctx.workflows.start({
  960. ...scripted("await parallel([() => agent('a'), () => agent('b')])\nreturn 'unreachable'"),
  961. parent,
  962. })
  963. await waitFor(() => { expect(provider.runs.length).toBe(2) })
  964. handle.cancel('user stop')
  965. const result = await handle.result
  966. expect(result.stopReason).toBe('cancelled')
  967. // The live worker reported both pairs itself; the ledger must not add
  968. // a synthesized duplicate on any path that settles inside the grace.
  969. expect(ends.map(end => end.outcome)).toEqual(['cancelled', 'cancelled'])
  970. expect(new Set(ends.map(end => end.seq)).size).toBe(2)
  971. expect(order.indexOf('run-end')).toBe(order.length - 1)
  972. await handle.dispose()
  973. })
  974. })
  975. describe('worker death', () => {
  976. it('a worker that exits before settling reports an error result and reaps its children', async () => {
  977. const ctx = new Context()
  978. await ctx.plugin(SubagentService)
  979. // The child's dispose() REJECTS on top of the worker death: the reap
  980. // must contain it (warn, not crash) while still emptying the registry.
  981. const cancelled: string[] = []
  982. const provider: SubagentProvider = {
  983. name: 'doomed',
  984. capabilities: { outputSchema: true, depthLimit: true, toolFilter: true, persona: false },
  985. inheritsParentContext: false,
  986. start: () => ({
  987. id: AgentId('doomed-child'),
  988. started: Promise.resolve(),
  989. result: new Promise(() => { /* never settles; the reap is the teardown */ }),
  990. cancel: (reason?: string) => { cancelled.push(reason ?? 'cancelled') },
  991. dispose: () => Promise.reject(new Error('dispose exploded during reap')),
  992. }),
  993. }
  994. ctx.subagents.registerProvider(provider)
  995. await ctx.plugin(WorkerWorkflowEngine, { provider: 'doomed', maxConcurrentAgents: 2 })
  996. const runEnds: WorkflowResultInfo[] = []
  997. ctx.on('workflow/end', (_info, result) => { runEnds.push(result) })
  998. const handle = ctx.workflows.start({
  999. // The stray child's start RPC reaches the host, then the script kills
  1000. // its own worker through the documented vm escape — the host must
  1001. // settle `error` with the exit diagnostics and wind the child down.
  1002. ...scripted(`
  1003. agent('doomed')
  1004. const proc = ${ESCAPE}
  1005. const st = globalThis.constructor.constructor('return setTimeout')()
  1006. await new Promise(resolve => st(resolve, 200))
  1007. proc.exit(7)
  1008. `),
  1009. parent: fakeParent(),
  1010. })
  1011. const result = await handle.result
  1012. expect(result.stopReason).toBe('error')
  1013. expect(result.error).toContain('exit code 7')
  1014. expect(result.agentsStarted).toBe(1)
  1015. // A worker death is a stop reason like any other: workflow/end fires
  1016. // with the error outcome — for a bus observer it is the only obituary.
  1017. expect(runEnds).toEqual([{ stopReason: 'error', error: result.error, agentsStarted: 1 }])
  1018. // Result already settled — this is the reap's promptness, not a
  1019. // cold-start race; tight explicit bound (see the helper's doc comment).
  1020. await waitFor(() => { expect(cancelled.length).toBe(1) }, 1000)
  1021. await handle.dispose()
  1022. }, 15_000)
  1023. it('an uncaught exception inside the worker surfaces as an error result and reaps the in-flight child', async () => {
  1024. const { ctx, parent, provider } = await setup({ manual: true })
  1025. const handle = ctx.workflows.start({
  1026. ...scripted(`
  1027. agent('in flight when the worker dies')
  1028. const proc = ${ESCAPE}
  1029. const st = globalThis.constructor.constructor('return setTimeout')()
  1030. await new Promise(resolve => st(resolve, 200))
  1031. proc.nextTick(() => { throw new Error('worker blew up') })
  1032. await new Promise(() => {})
  1033. `),
  1034. parent,
  1035. })
  1036. const result = await handle.result
  1037. expect(result.stopReason).toBe('error')
  1038. expect(result.error).toContain('worker blew up')
  1039. // The reap wound the stray child down (cancel + a CLEAN dispose).
  1040. // Result already settled — this is the reap's promptness, not a
  1041. // cold-start race; tight explicit bound (see the helper's doc comment).
  1042. await waitFor(() => {
  1043. expect(provider.runs.length).toBe(1)
  1044. expect(provider.runs[0]!.disposed).toBe(true)
  1045. }, 1000)
  1046. await handle.dispose()
  1047. }, 15_000)
  1048. it('a worker death pairs every stranded start: the synthesized cancelled agent-end precedes the error workflow/end', async () => {
  1049. const { ctx, parent, provider } = await setup({ manual: true })
  1050. const ends: { seq: number; outcome: string }[] = []
  1051. const order: string[] = []
  1052. ctx.on('workflow/agent-start', (_info, agent) => { order.push(`start:${agent.seq}`) })
  1053. ctx.on('workflow/agent-end', (_info, agent) => {
  1054. ends.push({ seq: agent.seq, outcome: agent.outcome })
  1055. order.push(`end:${agent.seq}`)
  1056. })
  1057. ctx.on('workflow/end', () => { order.push('run-end') })
  1058. const handle = ctx.workflows.start({
  1059. // Same choreography as the force-settle pairing test, but the worker
  1060. // DIES (the documented vm escape) instead of being terminated: the
  1061. // exit path must close slow's pair from the ledger too. The escaped
  1062. // setTimeout lets the already-posted messages flush before the kill.
  1063. ...scripted(`
  1064. const p = agent('slow')
  1065. await agent('fast')
  1066. const proc = ${ESCAPE}
  1067. const st = globalThis.constructor.constructor('return setTimeout')()
  1068. await new Promise(resolve => st(resolve, 150))
  1069. proc.exit(7)
  1070. `),
  1071. parent,
  1072. })
  1073. await waitFor(() => { expect(order.filter(entry => entry.startsWith('start:')).length).toBe(2) })
  1074. const fast = provider.runs.find(run => (run.request.prompt[0] as { text?: string }).text === 'fast')!
  1075. fast.settle(text('fast done'))
  1076. const result = await handle.result
  1077. expect(result.stopReason).toBe('error')
  1078. expect(result.error).toContain('exit code 7')
  1079. expect(ends).toEqual([
  1080. { seq: 2, outcome: 'completed' },
  1081. { seq: 1, outcome: 'cancelled' },
  1082. ])
  1083. expect(order.indexOf('run-end')).toBe(order.length - 1)
  1084. await handle.dispose()
  1085. }, 15_000)
  1086. it('a dispose ack racing the worker death is dropped, not crashed (post after exit)', async () => {
  1087. // Slow child disposal: the ack resolves only AFTER the worker died, so
  1088. // it has nowhere to go and must be dropped silently (the workerGone
  1089. // guard in post()).
  1090. const { ctx, parent, provider } = await setup({ disposeDelayMs: 300 })
  1091. const handle = ctx.workflows.start({
  1092. // The STRAY child settles instantly, so its wrapper starts the slow
  1093. // host-side disposal concurrently while the script goes on to kill
  1094. // its own worker — the ack then resolves into a dead thread.
  1095. ...scripted(`
  1096. agent('stray, never awaited')
  1097. const proc = ${ESCAPE}
  1098. const st = globalThis.constructor.constructor('return setTimeout')()
  1099. await new Promise(resolve => st(resolve, 150))
  1100. proc.exit(5)
  1101. `),
  1102. parent,
  1103. })
  1104. const result = await handle.result
  1105. expect(result.stopReason).toBe('error')
  1106. expect(result.error).toContain('exit code 5')
  1107. // Result already settled — this is the reap's promptness (bounded
  1108. // above the mock's fixed 300ms dispose delay, not a cold-start race);
  1109. // tight explicit bound (see the helper's doc comment).
  1110. await waitFor(() => { expect(provider.runs[0]!.disposed).toBe(true) }, 1000)
  1111. await handle.dispose()
  1112. }, 15_000)
  1113. it('a worker death AFTER a cancel reports cancelled, not error', async () => {
  1114. const { ctx, parent } = await setup({ config: { provider: 'stub', disposeGraceMs: 60_000 } })
  1115. const handle = ctx.workflows.start({
  1116. ...scripted(`
  1117. const proc = ${ESCAPE}
  1118. const st = globalThis.constructor.constructor('return setTimeout')()
  1119. log('armed')
  1120. await new Promise(resolve => st(resolve, 400))
  1121. proc.exit(3)
  1122. `),
  1123. parent,
  1124. })
  1125. const logs: string[] = []
  1126. ctx.on('workflow/log', (_info, message) => { logs.push(message) })
  1127. await waitFor(() => { expect(logs).toContain('armed') })
  1128. handle.cancel('stop it')
  1129. // The grace is deliberately huge: only the worker's own death (exit 3,
  1130. // unreachable by the cancel — the script ignores hooks) settles this.
  1131. const result = await handle.result
  1132. expect(result.stopReason).toBe('cancelled')
  1133. expect(result.error).toContain('stop it')
  1134. await handle.dispose()
  1135. }, 15_000)
  1136. })
  1137. describe('service surface', () => {
  1138. it('run ids are unique per start; the run handle and event payloads hold SEPARATE meta clones', async () => {
  1139. const { ctx, parent } = await setup()
  1140. let eventMeta: WorkflowRunInfo | undefined
  1141. ctx.on('workflow/start', (info) => { eventMeta = info })
  1142. const first = ctx.workflows.start({ ...scripted('return 1'), parent })
  1143. const second = ctx.workflows.start({ ...scripted('return 2'), parent })
  1144. expect(first.id).not.toBe(second.id)
  1145. eventMeta!.meta.name = 'corrupted'
  1146. expect(second.meta.name).toBe('test-flow')
  1147. await Promise.all([first.result, second.result])
  1148. await first.dispose()
  1149. await second.dispose()
  1150. })
  1151. it('unregisters ctx.workflows when the engine fiber is disposed (HMR safety), and default config runs (auto concurrency)', async () => {
  1152. const ctx = new Context()
  1153. await ctx.plugin(SubagentService)
  1154. const fiber = await ctx.plugin(WorkerWorkflowEngine, {})
  1155. expect(ctx.get('workflows')).toBeDefined()
  1156. // A zero-agent run through the DEFAULT config exercises the auto
  1157. // concurrency resolution (cores - 2, capped) in start().
  1158. const result = await run(ctx, fakeParent(), scripted('return 6 * 7'))
  1159. expect(result.value).toBe(42)
  1160. await fiber.dispose()
  1161. expect(ctx.get('workflows')).toBeUndefined()
  1162. })
  1163. it('has the class-plugin export shape (default = the engine service class)', () => {
  1164. expect(workerEngineModule.default).toBe(WorkerWorkflowEngine)
  1165. const loader = Object.create(Loader.prototype) as Loader
  1166. const unwrapped: unknown = loader.unwrapExports(workerEngineModule)
  1167. expect(unwrapped).toBe(WorkerWorkflowEngine)
  1168. })
  1169. })
  1170. })