workflow-workerthread.spec.ts 43 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906
  1. import { describe, expect, it, vi } from 'vitest'
  2. import { fileURLToPath } from 'node:url'
  3. import { Context } from 'cordis'
  4. import Loader from '@cordisjs/plugin-loader'
  5. import { AgentId } from '@deepseek-ai/dsh-agent'
  6. import type { Agent } from '@deepseek-ai/dsh-agent'
  7. import SubagentService from '@deepseek-ai/dsh-subagent'
  8. import type { SubagentCapabilities, SubagentProvider, SubagentResult, SubagentRun, SubagentStartRequest } from '@deepseek-ai/dsh-subagent'
  9. import type { WorkflowMeta, WorkflowResult, WorkflowResultInfo, WorkflowRunInfo } from '@deepseek-ai/dsh-workflow'
  10. import * as workerEngineModule from '../src/index.ts'
  11. import WorkerWorkflowEngine, { type Config } from '../src/index.ts'
  12. /** A minimal parent stand-in: the engine only threads it through to the provider. */
  13. function fakeParent(): Agent {
  14. return { id: AgentId('workflow-parent'), options: {} } as unknown as Agent
  15. }
  16. /** The vm-context escape hatch, spelled once: real Worker tests use it to make the WORKER misbehave. */
  17. const ESCAPE = "globalThis.constructor.constructor('return process')()"
  18. /** Linux coverage workers can take longer than Vitest's default waitFor timeout to start executing script. */
  19. const WORKER_CHILD_START_TIMEOUT_MS = 3_000
  20. /** One controllable child run: the test (or auto mode) settles it. */
  21. interface ControlledRun {
  22. request: SubagentStartRequest
  23. settle(result: SubagentResult): void
  24. cancelled: string | undefined
  25. disposed: boolean
  26. disposeCalls: number
  27. }
  28. /**
  29. * A scripted in-test provider over the REAL SubagentService registry: `auto`
  30. * settles each run via the reply function on a microtask; `manual` piles runs
  31. * up in `runs` for the test to settle. A run aborts (settles `aborted`) when
  32. * the request signal fires, like the real in-process backends.
  33. */
  34. class StubProvider implements SubagentProvider {
  35. readonly capabilities: SubagentCapabilities = { outputSchema: true, depthLimit: true, toolFilter: true }
  36. readonly inheritsParentContext = false
  37. readonly runs: ControlledRun[] = []
  38. constructor(
  39. readonly name: string,
  40. private readonly reply?: (request: SubagentStartRequest, index: number) => SubagentResult,
  41. private readonly disposeDelayMs = 0,
  42. ) {}
  43. start(request: SubagentStartRequest): SubagentRun {
  44. let settle!: (result: SubagentResult) => void
  45. const result = new Promise<SubagentResult>((resolve) => { settle = resolve })
  46. const controlled: ControlledRun = { request, settle, cancelled: undefined, disposed: false, disposeCalls: 0 }
  47. this.runs.push(controlled)
  48. const index = this.runs.length - 1
  49. request.signal?.addEventListener('abort', () => { settle({ output: [], stopReason: 'aborted' }) }, { once: true })
  50. if (this.reply) {
  51. const reply = this.reply
  52. queueMicrotask(() => { settle(reply(request, index)) })
  53. }
  54. return {
  55. id: AgentId(`stub-child-${index}`),
  56. result,
  57. cancel: (reason?: string) => {
  58. controlled.cancelled = reason ?? 'cancelled'
  59. settle({ output: [], stopReason: 'aborted' })
  60. },
  61. dispose: () => {
  62. controlled.disposeCalls += 1
  63. if (this.disposeDelayMs === 0) {
  64. controlled.disposed = true
  65. return Promise.resolve()
  66. }
  67. return new Promise<void>((resolve) => {
  68. setTimeout(() => {
  69. controlled.disposed = true
  70. resolve()
  71. }, this.disposeDelayMs)
  72. })
  73. },
  74. }
  75. }
  76. }
  77. /** Text-reply helper for auto providers. */
  78. function text(reply: string): SubagentResult {
  79. return { output: [{ type: 'text', text: reply }], stopReason: 'completed' }
  80. }
  81. interface SetupOptions {
  82. config?: Config
  83. reply?: (request: SubagentStartRequest, index: number) => SubagentResult
  84. manual?: boolean
  85. disposeDelayMs?: number
  86. }
  87. async function setup(options?: SetupOptions) {
  88. const ctx = new Context()
  89. await ctx.plugin(SubagentService)
  90. const provider = new StubProvider(
  91. 'stub',
  92. options?.manual ? undefined : options?.reply ?? (() => text('stub reply')),
  93. options?.disposeDelayMs ?? 0,
  94. )
  95. ctx.subagents.registerProvider(provider)
  96. // A fixed concurrency ceiling: the auto-resolved default is machine-derived
  97. // (cores - 2, floored at 1), so tests that expect N children in flight
  98. // would wedge on small CI runners.
  99. await ctx.plugin(WorkerWorkflowEngine, { provider: 'stub', maxConcurrentAgents: 8, ...options?.config })
  100. return { ctx, provider, parent: fakeParent() }
  101. }
  102. /** The standard test meta plus a body, spread into a start request. */
  103. function scripted(body: string, metaExtra?: Partial<WorkflowMeta>): { script: string; meta: WorkflowMeta } {
  104. return { script: body, meta: { name: 'test-flow', description: 'a test workflow', ...metaExtra } }
  105. }
  106. /** Start + await one run, disposing on the way out. */
  107. async function run(ctx: Context, parent: Agent, source: { script: string; meta: WorkflowMeta }, args?: unknown): Promise<WorkflowResult> {
  108. const handle = ctx.workflows.start({ ...source, parent, ...args !== undefined ? { args } : {} })
  109. try {
  110. return await handle.result
  111. } finally {
  112. await handle.dispose()
  113. }
  114. }
  115. describe('dsh-workflow-workerthread', () => {
  116. describe('script execution over a real worker thread', () => {
  117. it('runs a script end-to-end: agent() text results, phases, log, args, return value, events', async () => {
  118. const { ctx, parent, provider } = await setup({ reply: (_request, index) => text(`answer-${index}`) })
  119. const events: [string, unknown[]][] = []
  120. for (const name of ['workflow/start', 'workflow/phase', 'workflow/log', 'workflow/agent-start', 'workflow/agent-end', 'workflow/end'] as const) {
  121. ctx.on(name, (...payload: unknown[]) => { events.push([name, payload]) })
  122. }
  123. const result = await run(ctx, parent, scripted(`
  124. phase('Scan')
  125. log('starting with ' + args.files.length + ' files')
  126. const answers = await pipeline(args.files, (prev, item) => agent('read ' + item))
  127. phase('Report')
  128. return { answers, count: args.files.length }
  129. `, { phases: [{ title: 'Scan' }, { title: 'Report' }] }), { files: ['a.ts', 'b.ts'] })
  130. expect(result.stopReason).toBe('completed')
  131. expect(result.agentsStarted).toBe(2)
  132. expect(result.value).toEqual({ answers: ['answer-0', 'answer-1'], count: 2 })
  133. expect(provider.runs.every(r => r.disposed)).toBe(true)
  134. const names = events.map(([name]) => name)
  135. expect(names[0]).toBe('workflow/start')
  136. expect(names).toContain('workflow/phase')
  137. expect(names).toContain('workflow/log')
  138. expect(names.at(-1)).toBe('workflow/end')
  139. const info = events[0]![1][0] as WorkflowRunInfo
  140. expect(info.meta.name).toBe('test-flow')
  141. const end = events.at(-1)![1][1] as Record<string, unknown>
  142. expect(end).toEqual({ stopReason: 'completed', agentsStarted: 2 })
  143. expect('value' in end).toBe(false)
  144. })
  145. it('agent({schema, model}) forwards outputSchema and agentOptions to the provider across the thread', async () => {
  146. const { ctx, parent, provider } = await setup({
  147. reply: () => ({ output: [], structured: { files: ['x.ts', 'y.ts'] }, stopReason: 'completed' }),
  148. })
  149. const result = await run(ctx, parent, scripted(`
  150. const found = await agent('list files', { model: 'deepseek-v4-pro', schema: { type: 'object', properties: { files: { type: 'array', items: { type: 'string' } } }, required: ['files'] } })
  151. return { first: found.files[0], count: found.files.length }
  152. `))
  153. expect(result.value).toEqual({ first: 'x.ts', count: 2 })
  154. expect(provider.runs[0]!.request.outputSchema).toEqual({
  155. type: 'object',
  156. properties: { files: { type: 'array', items: { type: 'string' } } },
  157. required: ['files'],
  158. })
  159. expect(provider.runs[0]!.request.agentOptions).toEqual({ model: 'deepseek-v4-pro' })
  160. expect(provider.runs[0]!.request.parent).toBeDefined()
  161. })
  162. it('a fatal hook error inside the worker kills the script and reports the error', async () => {
  163. const { ctx, parent } = await setup()
  164. const result = await run(ctx, parent, scripted("return await parallel([() => agent('x', { isolation: 'worktree' })])"))
  165. expect(result.stopReason).toBe('error')
  166. expect(result.error).toContain('"isolation" is deferred')
  167. })
  168. it('a provider start failure crosses back as a fatal AGENT_START error', async () => {
  169. const { ctx, parent } = await setup({ config: { provider: 'nonexistent' } })
  170. const result = await run(ctx, parent, scripted("return await pipeline([1], () => agent('p'))"))
  171. expect(result.stopReason).toBe('error')
  172. expect(result.error).toContain('agent() could not start a child')
  173. })
  174. it('a child result REJECTION crosses back as a fatal AGENT_RESULT error (a broken provider is not a failed child)', async () => {
  175. const ctx = new Context()
  176. await ctx.plugin(SubagentService)
  177. const provider: SubagentProvider = {
  178. name: 'rejecting',
  179. capabilities: { outputSchema: true, depthLimit: true, toolFilter: true },
  180. inheritsParentContext: false,
  181. start: () => ({
  182. id: AgentId('reject-child'),
  183. result: Promise.reject(new Error('backend exploded')),
  184. cancel: () => { /* nothing in flight */ },
  185. dispose: () => Promise.resolve(),
  186. }),
  187. }
  188. ctx.subagents.registerProvider(provider)
  189. await ctx.plugin(WorkerWorkflowEngine, { provider: 'rejecting', maxConcurrentAgents: 2 })
  190. const result = await run(ctx, fakeParent(), scripted(`
  191. try { await agent('p'); return 'unreachable' } catch (e) { return { name: e.name, code: e.code, fatal: e.fatal, message: e.message } }
  192. `))
  193. expect(result.value).toMatchObject({ name: 'WorkflowError', code: 'AGENT_RESULT', fatal: true })
  194. expect((result.value as { message: string }).message).toContain('backend exploded')
  195. })
  196. it('a child whose dispose() rejects cannot wedge the script (the host acks anyway)', async () => {
  197. const ctx = new Context()
  198. await ctx.plugin(SubagentService)
  199. const provider: SubagentProvider = {
  200. name: 'bad-dispose',
  201. capabilities: { outputSchema: true, depthLimit: true, toolFilter: true },
  202. inheritsParentContext: false,
  203. start: () => ({
  204. id: AgentId('bad-dispose-child'),
  205. result: Promise.resolve({ output: [{ type: 'text', text: 'fine' }], stopReason: 'completed' }),
  206. cancel: () => { /* settled already */ },
  207. dispose: () => Promise.reject(new Error('dispose exploded')),
  208. }),
  209. }
  210. ctx.subagents.registerProvider(provider)
  211. await ctx.plugin(WorkerWorkflowEngine, { provider: 'bad-dispose', maxConcurrentAgents: 2 })
  212. const result = await run(ctx, fakeParent(), scripted("return await agent('p')"))
  213. expect(result.stopReason).toBe('completed')
  214. expect(result.value).toBe('fine')
  215. })
  216. it('a child dispose() rejecting an UNRENDERABLE value still acks — the containment warn is total', async () => {
  217. const ctx = new Context()
  218. await ctx.plugin(SubagentService)
  219. const provider: SubagentProvider = {
  220. name: 'coercion-trap-dispose',
  221. capabilities: { outputSchema: true, depthLimit: true, toolFilter: true },
  222. inheritsParentContext: false,
  223. start: () => ({
  224. id: AgentId('trap-child'),
  225. result: Promise.resolve({ output: [{ type: 'text', text: 'fine' }], stopReason: 'completed' }),
  226. cancel: () => { /* settled already */ },
  227. // The rejection VALUE's own coercion throws: a warn built with bare
  228. // String(error) would itself throw, skipping the ChildDisposed ack
  229. // and wedging the script's finally until the grace/terminate path.
  230. // eslint-disable-next-line @typescript-eslint/prefer-promise-reject-errors -- the non-Error rejection IS the scenario under test
  231. dispose: () => Promise.reject({ toString: () => { throw new Error('coercion trap') } }),
  232. }),
  233. }
  234. ctx.subagents.registerProvider(provider)
  235. await ctx.plugin(WorkerWorkflowEngine, { provider: 'coercion-trap-dispose', maxConcurrentAgents: 2 })
  236. const result = await run(ctx, fakeParent(), scripted("return await agent('p')"))
  237. expect(result.stopReason).toBe('completed')
  238. expect(result.value).toBe('fine')
  239. })
  240. it('the worker spawns with an EMPTY environment: an escaped script finds no ambient credentials', async () => {
  241. const { ctx, parent } = await setup()
  242. // A canary in the HARNESS process's env: with an inherited environment
  243. // the escape below would read it back (exactly how DEEPSEEK_API_KEY
  244. // would leak); env: {} in the spawn options is what keeps it out.
  245. process.env.WORKFLOW_ENV_CANARY = 'leak me'
  246. try {
  247. const result = await run(ctx, parent, scripted(`
  248. const proc = ${ESCAPE}
  249. return { canary: proc.env.WORKFLOW_ENV_CANARY ?? null, keys: Object.keys(proc.env).length }
  250. `))
  251. expect(result.stopReason).toBe('completed')
  252. expect(result.value).toEqual({ canary: null, keys: 0 })
  253. } finally {
  254. delete process.env.WORKFLOW_ENV_CANARY
  255. }
  256. })
  257. it('the unbuilt worker forwards exactly TSX_TSCONFIG_PATH through the scrub: the paths-map pin survives, secrets do not', async () => {
  258. const { ctx, parent } = await setup()
  259. // The ACP snapshot harness runs the parent with its cwd OUTSIDE the
  260. // repo and pins the repo tsconfig through this variable; the worker
  261. // must inherit the pin (or its dsh-* imports silently resolve to
  262. // unbuilt lib/ bundles) while every other variable stays scrubbed.
  263. const tsconfig = fileURLToPath(new URL('../../../../tsconfig.json', import.meta.url))
  264. process.env.TSX_TSCONFIG_PATH = tsconfig
  265. process.env.WORKFLOW_ENV_CANARY = 'leak me'
  266. try {
  267. const result = await run(ctx, parent, scripted(`
  268. const proc = ${ESCAPE}
  269. return { keys: Object.keys(proc.env), tsconfig: proc.env.TSX_TSCONFIG_PATH }
  270. `))
  271. expect(result.stopReason).toBe('completed')
  272. expect(result.value).toEqual({ keys: ['TSX_TSCONFIG_PATH'], tsconfig })
  273. } finally {
  274. delete process.env.TSX_TSCONFIG_PATH
  275. delete process.env.WORKFLOW_ENV_CANARY
  276. }
  277. })
  278. })
  279. describe('lifecycle: parse errors, cancellation, termination, disposal', () => {
  280. it('start() throws synchronously for invalid meta data or an unparseable body (host-side pre-checks)', async () => {
  281. const { ctx, parent } = await setup()
  282. // Meta is DATA — shape violations reject loud, every one named.
  283. expect(() => ctx.workflows.start({ script: 'return 1', meta: { name: '', description: 'd' }, parent })).toThrow(/meta\.name must be a non-empty string/)
  284. expect(() => ctx.workflows.start({ script: 'return 1', meta: { name: 'x', description: 'd', extra: 1 } as unknown as WorkflowMeta, parent })).toThrow(/META_INVALID|not a recognized field/)
  285. expect(() => ctx.workflows.start({ ...scripted('return ((('), parent })).toThrow(/does not parse/)
  286. // The likeliest authoring slip — a Claude Code-style meta header in the
  287. // body — gets a pointed message, not a bare SyntaxError.
  288. expect(() => ctx.workflows.start({ ...scripted("export const meta = { name: 'x', description: 'd' }\nreturn 1"), parent })).toThrow(/meta rides the `meta` request field/)
  289. })
  290. it('cancel() aborts in-flight children (signal AND cancel RPC) and settles the run cancelled', async () => {
  291. const { ctx, parent, provider } = await setup({ manual: true })
  292. const ends: unknown[] = []
  293. ctx.on('workflow/agent-end', (_info, agent) => { ends.push(agent) })
  294. const runEnds: WorkflowResultInfo[] = []
  295. ctx.on('workflow/end', (_info, result) => { runEnds.push(result) })
  296. const handle = ctx.workflows.start({ ...scripted("return await agent('long job')"), parent })
  297. await vi.waitFor(() => { expect(provider.runs.length).toBe(1) })
  298. handle.cancel('user stopped it')
  299. const result = await handle.result
  300. expect(result.stopReason).toBe('cancelled')
  301. expect(result.error).toContain('user stopped it')
  302. await handle.dispose()
  303. expect(provider.runs[0]!.disposed).toBe(true)
  304. expect(ends).toEqual([expect.objectContaining({ seq: 1, outcome: 'cancelled' })])
  305. // workflow/end is an observer's only death signal: it fires for a
  306. // cancelled run too, mirroring the settled outcome data.
  307. expect(runEnds).toEqual([{ stopReason: 'cancelled', error: result.error, agentsStarted: result.agentsStarted }])
  308. })
  309. it('an already-aborted request signal cancels before the body ever runs (the go handshake holds it)', async () => {
  310. const { ctx, parent, provider } = await setup()
  311. const controller = new AbortController()
  312. controller.abort()
  313. const logs: string[] = []
  314. ctx.on('workflow/log', (_info, message) => { logs.push(message) })
  315. const handle = ctx.workflows.start({ ...scripted("log('ran')\nreturn 123"), parent, signal: controller.signal })
  316. const result = await handle.result
  317. expect(result.stopReason).toBe('cancelled')
  318. expect(result.value).toBeNull()
  319. expect(logs).toEqual([])
  320. expect(provider.runs.length).toBe(0)
  321. await handle.dispose()
  322. })
  323. it('cancel() right after start() cancels before the body runs; the signal aborting mid-run cancels like cancel()', async () => {
  324. const { ctx, parent, provider } = await setup({ manual: true })
  325. const first = ctx.workflows.start({ ...scripted("return await agent('never')"), parent })
  326. // No-reason cancel: the canonical default reason must ride the result.
  327. first.cancel()
  328. const firstResult = await first.result
  329. expect(firstResult.stopReason).toBe('cancelled')
  330. expect(firstResult.error).toContain('workflow cancelled')
  331. expect(provider.runs.length).toBe(0)
  332. await first.dispose()
  333. const controller = new AbortController()
  334. const second = ctx.workflows.start({ ...scripted("return await agent('job')"), parent, signal: controller.signal })
  335. await vi.waitFor(() => { expect(provider.runs.length).toBe(1) })
  336. controller.abort()
  337. expect((await second.result).stopReason).toBe('cancelled')
  338. await second.dispose()
  339. })
  340. it('a child-start racing the host cancel is refused: no child starts after cancellation', async () => {
  341. const { ctx, parent, provider } = await setup({ manual: true })
  342. // Cancel from INSIDE the log listener: the worker has already posted
  343. // its child-start (queued right behind the log message), so the host
  344. // processes it with cancelReason set — the refusal arm no real-world
  345. // timing can hit reliably. (The closure runs only after `handle` below
  346. // is initialized — the listener fires on the worker's first message.)
  347. ctx.on('workflow/log', () => { handle.cancel('cancelled from the log listener') })
  348. const handle = ctx.workflows.start({ ...scripted("log('mark')\nreturn await agent('late')"), parent })
  349. const result = await handle.result
  350. expect(result.stopReason).toBe('cancelled')
  351. expect(provider.runs.length).toBe(0)
  352. await handle.dispose()
  353. })
  354. it('post-cancel narration is suppressed host-side, and completion racing a cancel reports cancelled', async () => {
  355. const { ctx, parent } = await setup()
  356. const narration: string[] = []
  357. ctx.on('workflow/log', (_info, message) => { narration.push(message) })
  358. ctx.on('workflow/phase', (_info, title) => { narration.push(`phase:${title}`) })
  359. const handle = ctx.workflows.start({
  360. // The sync spin keeps the worker's loop busy so the cancel message
  361. // cannot be processed before the script settles `completed` — the
  362. // worker posts a completed result that must LOSE to the in-flight
  363. // host cancellation. The trailing narration exercises host-side
  364. // suppression: posted pre-cancel-processing worker-side, arriving
  365. // post-cancel host-side.
  366. ...scripted(`
  367. log('started')
  368. const end = Date.now() + 1000
  369. while (Date.now() < end) {}
  370. phase('late phase')
  371. log('late log')
  372. return 'done'
  373. `),
  374. parent,
  375. })
  376. await vi.waitFor(() => { expect(narration).toContain('started') })
  377. handle.cancel('raced the completion')
  378. const result = await handle.result
  379. expect(result.stopReason).toBe('cancelled')
  380. expect(result.error).toContain('raced the completion')
  381. expect(narration).toEqual(['started'])
  382. await handle.dispose()
  383. }, 15_000)
  384. it('cancel() force-settles a script parked on a promise no hook owns, and TERMINATES its worker', async () => {
  385. const { ctx, parent } = await setup({ config: { provider: 'stub', disposeGraceMs: 50 } })
  386. const runEnds: WorkflowResultInfo[] = []
  387. ctx.on('workflow/end', (_info, result) => { runEnds.push(result) })
  388. const handle = ctx.workflows.start({
  389. ...scripted("await new Promise(() => {})\nreturn 'unreachable'"),
  390. parent,
  391. })
  392. handle.cancel('user aborted')
  393. const result = await handle.result
  394. expect(result.stopReason).toBe('cancelled')
  395. expect(result.error).toContain('user aborted')
  396. // The grace force-settle fires workflow/end exactly like an ordinary
  397. // settlement — a terminated script's death still reaches observers.
  398. expect(runEnds).toEqual([{ stopReason: 'cancelled', error: result.error, agentsStarted: 0 }])
  399. await handle.dispose()
  400. })
  401. it('dispose() on a stuck script returns within the grace instead of hanging (result settles cancelled)', async () => {
  402. const { ctx, parent } = await setup({ config: { provider: 'stub', disposeGraceMs: 50 } })
  403. const handle = ctx.workflows.start({
  404. ...scripted("await new Promise(() => {})\nreturn 'unreachable'"),
  405. parent,
  406. })
  407. const before = Date.now()
  408. await handle.dispose()
  409. expect(Date.now() - before).toBeLessThan(2000)
  410. const result = await handle.result
  411. expect(result.stopReason).toBe('cancelled')
  412. })
  413. it('dispose() is idempotent and settles cleanly after a completed run', async () => {
  414. const { ctx, parent } = await setup()
  415. const handle = ctx.workflows.start({ ...scripted('return 1'), parent })
  416. await handle.result
  417. await handle.dispose()
  418. await handle.dispose()
  419. })
  420. it('a settled run arms NO grace timer: disposing a completed run must not pin it for disposeGraceMs', async () => {
  421. // A distinctive grace so the spy can tell the cancel-path grace timer
  422. // apart from every other timeout in flight.
  423. const GRACE = 44_444
  424. const { ctx, parent } = await setup({ config: { provider: 'stub', disposeGraceMs: GRACE } })
  425. const handle = ctx.workflows.start({ ...scripted('return 1'), parent })
  426. await handle.result
  427. const spy = vi.spyOn(globalThis, 'setTimeout')
  428. try {
  429. await handle.dispose()
  430. // dispose()'s own bounded-wait sleep is the ONLY grace-sized timer
  431. // allowed here; before the settled guard, cancel() armed a second one
  432. // that nothing would ever clear (the run was already settled), keeping
  433. // the WorkerRun/Worker closure alive until the grace expired.
  434. const graceTimers = spy.mock.calls.filter(call => call[1] === GRACE)
  435. expect(graceTimers.length).toBe(1)
  436. } finally {
  437. spy.mockRestore()
  438. }
  439. })
  440. it('strays: children fired without await are aborted once the script settles, and dispose() waits for their disposal', async () => {
  441. const { ctx, parent, provider } = await setup({ manual: true, disposeDelayMs: 40 })
  442. const handle = ctx.workflows.start({
  443. ...scripted(`
  444. agent('stray')
  445. return 'done without awaiting'
  446. `),
  447. parent,
  448. })
  449. const result = await handle.result
  450. expect(result.stopReason).toBe('completed')
  451. await vi.waitFor(() => { expect(provider.runs.length).toBe(1) })
  452. await handle.dispose()
  453. // Not a waitFor: by the time dispose() returns, the slow child disposal
  454. // must already be complete (host-side registry quiescence).
  455. expect(provider.runs[0]!.disposed).toBe(true)
  456. })
  457. it('the settle-reap fires the request signal too: a provider honoring ONLY the signal winds its stray down promptly', async () => {
  458. const ctx = new Context()
  459. await ctx.plugin(SubagentService)
  460. const aborted: string[] = []
  461. const provider: SubagentProvider = {
  462. name: 'signal-only',
  463. capabilities: { outputSchema: true, depthLimit: true, toolFilter: true },
  464. inheritsParentContext: false,
  465. start: (request) => {
  466. let settle!: (result: SubagentResult) => void
  467. const result = new Promise<SubagentResult>((resolve) => { settle = resolve })
  468. request.signal?.addEventListener('abort', () => {
  469. aborted.push(String(request.signal?.reason))
  470. settle({ output: [], stopReason: 'aborted' })
  471. }, { once: true })
  472. return {
  473. id: AgentId('signal-only-child'),
  474. result,
  475. // The seam leaves a provider free to honor EITHER cancel channel;
  476. // this one deliberately ignores run.cancel() — only the request
  477. // signal can wind it down.
  478. cancel: () => { /* signal-only by design */ },
  479. dispose: () => Promise.resolve(),
  480. }
  481. },
  482. }
  483. ctx.subagents.registerProvider(provider)
  484. await ctx.plugin(WorkerWorkflowEngine, { provider: 'signal-only', maxConcurrentAgents: 2 })
  485. const handle = ctx.workflows.start({
  486. ...scripted(`
  487. agent('stray, never awaited')
  488. return 'done'
  489. `),
  490. parent: fakeParent(),
  491. })
  492. const result = await handle.result
  493. expect(result.stopReason).toBe('completed')
  494. // BEFORE dispose(): the settlement itself must have aborted the signal —
  495. // without it this child would stay live until dispose's terminate.
  496. await vi.waitFor(() => { expect(aborted).toEqual(['workflow settled']) })
  497. await handle.dispose()
  498. })
  499. it("cancel() drives each child's explicit cancel() host-side: a wedged worker cannot delay it", async () => {
  500. const ctx = new Context()
  501. await ctx.plugin(SubagentService)
  502. let starts = 0
  503. const cancelled: string[] = []
  504. const provider: SubagentProvider = {
  505. name: 'cancel-only',
  506. capabilities: { outputSchema: true, depthLimit: true, toolFilter: true },
  507. inheritsParentContext: false,
  508. start: () => {
  509. starts += 1
  510. return {
  511. id: AgentId('cancel-only-child'),
  512. result: new Promise(() => { /* only cancel() ends this child */ }),
  513. // Deliberately ignores the request signal — the seam leaves a
  514. // provider free to honor ONLY the explicit cancel() channel.
  515. cancel: (reason?: string) => { cancelled.push(reason ?? 'cancelled') },
  516. dispose: () => Promise.resolve(),
  517. }
  518. },
  519. }
  520. ctx.subagents.registerProvider(provider)
  521. // A deliberately huge grace: if only the grace/terminate reap could
  522. // reach this child, the assertion below would time out first.
  523. await ctx.plugin(WorkerWorkflowEngine, { provider: 'cancel-only', maxConcurrentAgents: 2, disposeGraceMs: 30_000 })
  524. const handle = ctx.workflows.start({
  525. // The stray child's start RPC reaches the host, then the script wedges
  526. // its own worker in a synchronous spin: the worker cannot process the
  527. // Cancel message, so it can relay NO ChildCancel RPC — only the host's
  528. // own children loop can deliver the explicit cancel in time. The
  529. // microtask yields let the agent() continuation POST its child-start
  530. // before the spin seizes the worker's loop (the posted message needs
  531. // no further worker-loop turns to reach the host).
  532. ...scripted(`
  533. agent('wedged child')
  534. for (let i = 0; i < 20; i++) await null
  535. const end = Date.now() + 1500
  536. while (Date.now() < end) {}
  537. return 'raced'
  538. `),
  539. parent: fakeParent(),
  540. })
  541. await vi.waitFor(() => { expect(starts).toBe(1) }, { timeout: WORKER_CHILD_START_TIMEOUT_MS })
  542. handle.cancel('stop now')
  543. await vi.waitFor(() => { expect(cancelled).toEqual(['stop now']) }, { timeout: 800 })
  544. // The wedged worker's own completion loses to the in-flight cancel.
  545. const result = await handle.result
  546. expect(result.stopReason).toBe('cancelled')
  547. await handle.dispose()
  548. }, 15_000)
  549. it('dispose() on a wedged worker host-drives child disposal inside the grace: it returns with the children DISPOSED, not with their teardown still in flight', async () => {
  550. const { ctx, parent, provider } = await setup({
  551. manual: true,
  552. disposeDelayMs: 40,
  553. config: { provider: 'stub', maxConcurrentAgents: 8, disposeGraceMs: 400 },
  554. })
  555. const handle = ctx.workflows.start({
  556. // Same shape as the wedged-cancel test above: the child's start RPC
  557. // reaches the host, then the script seizes its worker's loop, so the
  558. // worker can relay NO dispose RPC — the host's own dispose() drive is
  559. // the only thing that can start (and finish) this child's disposal
  560. // before the grace runs out.
  561. ...scripted(`
  562. agent('wedged child')
  563. for (let i = 0; i < 20; i++) await null
  564. const end = Date.now() + 1500
  565. while (Date.now() < end) {}
  566. return 'raced'
  567. `),
  568. parent,
  569. })
  570. await vi.waitFor(() => { expect(provider.runs.length).toBe(1) }, { timeout: WORKER_CHILD_START_TIMEOUT_MS })
  571. const before = Date.now()
  572. await handle.dispose()
  573. // Bounded by the grace (plus the terminate), never by the 1.5s spin.
  574. expect(Date.now() - before).toBeLessThan(1200)
  575. // Not a waitFor: dispose() resolving IS the quiescence claim — the slow
  576. // child disposal must be complete, not merely started (before the
  577. // host-driven drive, disposal only STARTED at the post-terminate reap,
  578. // so dispose() returned with it still in flight).
  579. expect(provider.runs[0]!.disposed).toBe(true)
  580. const result = await handle.result
  581. expect(result.stopReason).toBe('cancelled')
  582. }, 15_000)
  583. it('a live child disposed by the dispose() drive is disposed ONCE, and the worker\'s late dispose RPC still gets its ack (the script settles, not the grace)', async () => {
  584. const { ctx, parent, provider } = await setup({ manual: true })
  585. const handle = ctx.workflows.start({
  586. ...scripted(`
  587. await agent('long child')
  588. return 'unreachable'
  589. `),
  590. parent,
  591. })
  592. await vi.waitFor(() => { expect(provider.runs.length).toBe(1) })
  593. const handleDispose = handle.dispose()
  594. const result = await handle.result
  595. // The script itself settled (the wrapper's own dispose RPC found the
  596. // child already reaped host-side and was acked) — a missing ack would
  597. // wedge the wrapper's finally until the 5s default grace force-settle.
  598. expect(result.stopReason).toBe('cancelled')
  599. expect(result.error).toContain('workflow disposed')
  600. await handleDispose
  601. expect(provider.runs[0]!.disposed).toBe(true)
  602. // The memo: the host drive and the worker's RPC share one disposal.
  603. expect(provider.runs[0]!.disposeCalls).toBe(1)
  604. })
  605. it('the grace force-settle pairs every stranded start: a host-synthesized cancelled agent-end lands before workflow/end', async () => {
  606. const { ctx, parent, provider } = await setup({ manual: true, config: { provider: 'stub', maxConcurrentAgents: 8, disposeGraceMs: 300 } })
  607. const ends: { seq: number; outcome: string }[] = []
  608. const order: string[] = []
  609. ctx.on('workflow/agent-start', (_info, agent) => { order.push(`start:${agent.seq}`) })
  610. ctx.on('workflow/agent-end', (_info, agent) => {
  611. ends.push({ seq: agent.seq, outcome: agent.outcome })
  612. order.push(`end:${agent.seq}`)
  613. })
  614. ctx.on('workflow/end', () => { order.push('run-end') })
  615. const handle = ctx.workflows.start({
  616. // 'slow' starts and its agent-start crosses to observers (the awaited
  617. // 'fast' call keeps the worker loop turning), then the script seizes
  618. // the loop: the wedged worker can never author slow's agent-end —
  619. // only the host's ledger can close the pair.
  620. ...scripted(`
  621. const p = agent('slow')
  622. await agent('fast')
  623. const end = Date.now() + 1500
  624. while (Date.now() < end) {}
  625. return 'raced'
  626. `),
  627. parent,
  628. })
  629. await vi.waitFor(() => { expect(order.filter(entry => entry.startsWith('start:')).length).toBe(2) })
  630. const fast = provider.runs.find(run => (run.request.prompt[0] as { text?: string }).text === 'fast')!
  631. fast.settle(text('fast done'))
  632. handle.cancel('stop now')
  633. const result = await handle.result
  634. expect(result.stopReason).toBe('cancelled')
  635. // fast's end is the worker's own report; slow's is host-synthesized at
  636. // the force-settle — exactly one end per started seq, no third event.
  637. expect(ends).toEqual([
  638. { seq: 2, outcome: 'completed' },
  639. { seq: 1, outcome: 'cancelled' },
  640. ])
  641. // Both ends reached observers BEFORE workflow/end: a progress consumer
  642. // can finalize its state at run-end without dangling agents.
  643. expect(order.indexOf('run-end')).toBe(order.length - 1)
  644. await handle.dispose()
  645. }, 15_000)
  646. it('graceful cancellation keeps pairing worker-authored: exactly one agent-end per start, nothing synthesized on top', async () => {
  647. const { ctx, parent, provider } = await setup({ manual: true })
  648. const ends: { seq: number; outcome: string }[] = []
  649. const order: string[] = []
  650. ctx.on('workflow/agent-end', (_info, agent) => {
  651. ends.push({ seq: agent.seq, outcome: agent.outcome })
  652. order.push(`end:${agent.seq}`)
  653. })
  654. ctx.on('workflow/end', () => { order.push('run-end') })
  655. const handle = ctx.workflows.start({
  656. ...scripted("await parallel([() => agent('a'), () => agent('b')])\nreturn 'unreachable'"),
  657. parent,
  658. })
  659. await vi.waitFor(() => { expect(provider.runs.length).toBe(2) })
  660. handle.cancel('user stop')
  661. const result = await handle.result
  662. expect(result.stopReason).toBe('cancelled')
  663. // The live worker reported both pairs itself; the ledger must not add
  664. // a synthesized duplicate on any path that settles inside the grace.
  665. expect(ends.map(end => end.outcome)).toEqual(['cancelled', 'cancelled'])
  666. expect(new Set(ends.map(end => end.seq)).size).toBe(2)
  667. expect(order.indexOf('run-end')).toBe(order.length - 1)
  668. await handle.dispose()
  669. })
  670. })
  671. describe('worker death', () => {
  672. it('a worker that exits before settling reports an error result and reaps its children', async () => {
  673. const ctx = new Context()
  674. await ctx.plugin(SubagentService)
  675. // The child's dispose() REJECTS on top of the worker death: the reap
  676. // must contain it (warn, not crash) while still emptying the registry.
  677. const cancelled: string[] = []
  678. const provider: SubagentProvider = {
  679. name: 'doomed',
  680. capabilities: { outputSchema: true, depthLimit: true, toolFilter: true },
  681. inheritsParentContext: false,
  682. start: () => ({
  683. id: AgentId('doomed-child'),
  684. result: new Promise(() => { /* never settles; the reap is the teardown */ }),
  685. cancel: (reason?: string) => { cancelled.push(reason ?? 'cancelled') },
  686. dispose: () => Promise.reject(new Error('dispose exploded during reap')),
  687. }),
  688. }
  689. ctx.subagents.registerProvider(provider)
  690. await ctx.plugin(WorkerWorkflowEngine, { provider: 'doomed', maxConcurrentAgents: 2 })
  691. const runEnds: WorkflowResultInfo[] = []
  692. ctx.on('workflow/end', (_info, result) => { runEnds.push(result) })
  693. const handle = ctx.workflows.start({
  694. // The stray child's start RPC reaches the host, then the script kills
  695. // its own worker through the documented vm escape — the host must
  696. // settle `error` with the exit diagnostics and wind the child down.
  697. ...scripted(`
  698. agent('doomed')
  699. const proc = ${ESCAPE}
  700. const st = globalThis.constructor.constructor('return setTimeout')()
  701. await new Promise(resolve => st(resolve, 200))
  702. proc.exit(7)
  703. `),
  704. parent: fakeParent(),
  705. })
  706. const result = await handle.result
  707. expect(result.stopReason).toBe('error')
  708. expect(result.error).toContain('exit code 7')
  709. expect(result.agentsStarted).toBe(1)
  710. // A worker death is a stop reason like any other: workflow/end fires
  711. // with the error outcome — for a bus observer it is the only obituary.
  712. expect(runEnds).toEqual([{ stopReason: 'error', error: result.error, agentsStarted: 1 }])
  713. await vi.waitFor(() => { expect(cancelled.length).toBe(1) })
  714. await handle.dispose()
  715. }, 15_000)
  716. it('an uncaught exception inside the worker surfaces as an error result and reaps the in-flight child', async () => {
  717. const { ctx, parent, provider } = await setup({ manual: true })
  718. const handle = ctx.workflows.start({
  719. ...scripted(`
  720. agent('in flight when the worker dies')
  721. const proc = ${ESCAPE}
  722. const st = globalThis.constructor.constructor('return setTimeout')()
  723. await new Promise(resolve => st(resolve, 200))
  724. proc.nextTick(() => { throw new Error('worker blew up') })
  725. await new Promise(() => {})
  726. `),
  727. parent,
  728. })
  729. const result = await handle.result
  730. expect(result.stopReason).toBe('error')
  731. expect(result.error).toContain('worker blew up')
  732. // The reap wound the stray child down (cancel + a CLEAN dispose).
  733. await vi.waitFor(() => {
  734. expect(provider.runs.length).toBe(1)
  735. expect(provider.runs[0]!.disposed).toBe(true)
  736. })
  737. await handle.dispose()
  738. }, 15_000)
  739. it('a worker death pairs every stranded start: the synthesized cancelled agent-end precedes the error workflow/end', async () => {
  740. const { ctx, parent, provider } = await setup({ manual: true })
  741. const ends: { seq: number; outcome: string }[] = []
  742. const order: string[] = []
  743. ctx.on('workflow/agent-start', (_info, agent) => { order.push(`start:${agent.seq}`) })
  744. ctx.on('workflow/agent-end', (_info, agent) => {
  745. ends.push({ seq: agent.seq, outcome: agent.outcome })
  746. order.push(`end:${agent.seq}`)
  747. })
  748. ctx.on('workflow/end', () => { order.push('run-end') })
  749. const handle = ctx.workflows.start({
  750. // Same choreography as the force-settle pairing test, but the worker
  751. // DIES (the documented vm escape) instead of being terminated: the
  752. // exit path must close slow's pair from the ledger too. The escaped
  753. // setTimeout lets the already-posted messages flush before the kill.
  754. ...scripted(`
  755. const p = agent('slow')
  756. await agent('fast')
  757. const proc = ${ESCAPE}
  758. const st = globalThis.constructor.constructor('return setTimeout')()
  759. await new Promise(resolve => st(resolve, 150))
  760. proc.exit(7)
  761. `),
  762. parent,
  763. })
  764. await vi.waitFor(() => { expect(order.filter(entry => entry.startsWith('start:')).length).toBe(2) })
  765. const fast = provider.runs.find(run => (run.request.prompt[0] as { text?: string }).text === 'fast')!
  766. fast.settle(text('fast done'))
  767. const result = await handle.result
  768. expect(result.stopReason).toBe('error')
  769. expect(result.error).toContain('exit code 7')
  770. expect(ends).toEqual([
  771. { seq: 2, outcome: 'completed' },
  772. { seq: 1, outcome: 'cancelled' },
  773. ])
  774. expect(order.indexOf('run-end')).toBe(order.length - 1)
  775. await handle.dispose()
  776. }, 15_000)
  777. it('a dispose ack racing the worker death is dropped, not crashed (post after exit)', async () => {
  778. // Slow child disposal: the ack resolves only AFTER the worker died, so
  779. // it has nowhere to go and must be dropped silently (the workerGone
  780. // guard in post()).
  781. const { ctx, parent, provider } = await setup({ disposeDelayMs: 300 })
  782. const handle = ctx.workflows.start({
  783. // The STRAY child settles instantly, so its wrapper starts the slow
  784. // host-side disposal concurrently while the script goes on to kill
  785. // its own worker — the ack then resolves into a dead thread.
  786. ...scripted(`
  787. agent('stray, never awaited')
  788. const proc = ${ESCAPE}
  789. const st = globalThis.constructor.constructor('return setTimeout')()
  790. await new Promise(resolve => st(resolve, 150))
  791. proc.exit(5)
  792. `),
  793. parent,
  794. })
  795. const result = await handle.result
  796. expect(result.stopReason).toBe('error')
  797. expect(result.error).toContain('exit code 5')
  798. await vi.waitFor(() => { expect(provider.runs[0]!.disposed).toBe(true) })
  799. await handle.dispose()
  800. }, 15_000)
  801. it('a worker death AFTER a cancel reports cancelled, not error', async () => {
  802. const { ctx, parent } = await setup({ config: { provider: 'stub', disposeGraceMs: 60_000 } })
  803. const handle = ctx.workflows.start({
  804. ...scripted(`
  805. const proc = ${ESCAPE}
  806. const st = globalThis.constructor.constructor('return setTimeout')()
  807. log('armed')
  808. await new Promise(resolve => st(resolve, 400))
  809. proc.exit(3)
  810. `),
  811. parent,
  812. })
  813. const logs: string[] = []
  814. ctx.on('workflow/log', (_info, message) => { logs.push(message) })
  815. await vi.waitFor(() => { expect(logs).toContain('armed') })
  816. handle.cancel('stop it')
  817. // The grace is deliberately huge: only the worker's own death (exit 3,
  818. // unreachable by the cancel — the script ignores hooks) settles this.
  819. const result = await handle.result
  820. expect(result.stopReason).toBe('cancelled')
  821. expect(result.error).toContain('stop it')
  822. await handle.dispose()
  823. }, 15_000)
  824. })
  825. describe('service surface', () => {
  826. it('run ids are unique per start; the run handle and event payloads hold SEPARATE meta clones', async () => {
  827. const { ctx, parent } = await setup()
  828. let eventMeta: WorkflowRunInfo | undefined
  829. ctx.on('workflow/start', (info) => { eventMeta = info })
  830. const first = ctx.workflows.start({ ...scripted('return 1'), parent })
  831. const second = ctx.workflows.start({ ...scripted('return 2'), parent })
  832. expect(first.id).not.toBe(second.id)
  833. eventMeta!.meta.name = 'corrupted'
  834. expect(second.meta.name).toBe('test-flow')
  835. await Promise.all([first.result, second.result])
  836. await first.dispose()
  837. await second.dispose()
  838. })
  839. it('unregisters ctx.workflows when the engine fiber is disposed (HMR safety), and default config runs (auto concurrency)', async () => {
  840. const ctx = new Context()
  841. await ctx.plugin(SubagentService)
  842. const fiber = await ctx.plugin(WorkerWorkflowEngine, {})
  843. expect(ctx.get('workflows')).toBeDefined()
  844. // A zero-agent run through the DEFAULT config exercises the auto
  845. // concurrency resolution (cores - 2, capped) in start().
  846. const result = await run(ctx, fakeParent(), scripted('return 6 * 7'))
  847. expect(result.value).toBe(42)
  848. await fiber.dispose()
  849. expect(ctx.get('workflows')).toBeUndefined()
  850. })
  851. it('has the class-plugin export shape (default = the engine service class)', () => {
  852. expect(workerEngineModule.default).toBe(WorkerWorkflowEngine)
  853. const loader = Object.create(Loader.prototype) as Loader
  854. const unwrapped: unknown = loader.unwrapExports(workerEngineModule)
  855. expect(unwrapped).toBe(WorkerWorkflowEngine)
  856. })
  857. })
  858. })