host.ts 32 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648
  1. /**
  2. * The host half of one worker-engine run: spawn the Worker, bridge its child
  3. * RPC onto the holder-bound subagent service, fan its observer messages into
  4. * the engine's events, and own cancellation, the settle-within-grace
  5. * guarantee, and child cleanup. The worker's lifetime IS the run's lifetime:
  6. * `dispose()` always ends with `worker.terminate()`, so no thread outlives its
  7. * run.
  8. *
  9. * The run's `result` promise settles exactly once, from whichever of these
  10. * lands first: receipt of the worker's `result` message, an unexpected worker
  11. * death (`error`/`messageerror`/premature `exit` → `stopReason: 'error'`, or
  12. * `'cancelled'` when a cancel was in flight), or the post-cancel grace timer (a
  13. * script that never settles is force-settled `cancelled` and its worker
  14. * terminated — the real kill an in-process engine could not perform). At
  15. * `result` receipt the host snapshots whether caller/signal/dispose
  16. * cancellation is already in flight: an earlier cancellation overrides a
  17. * non-cancelled report; otherwise the report wins before settlement-only child
  18. * cleanup invokes arbitrary provider callbacks. Worker death uses the same
  19. * boundary: it claims `error` (or a previously requested `cancelled`) before
  20. * reaping children, so cleanup callbacks cannot rewrite the outcome. That
  21. * first signal also closes inbound message admission: Node may emit `error`,
  22. * then deliver queued messages, then emit `exit`, but those late messages may
  23. * neither create work nor narrate after settlement. If Result or grace already
  24. * owns the outcome, death preserves it while still cleaning resources; the
  25. * eventual exit performs a final disposal-only sweep without repeating child
  26. * cancellation.
  27. *
  28. * Provider starts and published children are tracked separately. Every start
  29. * receives one shared per-run abort signal; the provider owns partial setup
  30. * until its promise fulfills. If admission closes while a start is pending,
  31. * the signal aborts it; a late fulfillment is disposed without publication to
  32. * the worker. Ready runs enter a callId registry whose memoized disposal is
  33. * shared by graceful worker RPC, public disposal, normal-settlement reap, and
  34. * worker-death cleanup. Quiescence requires both pending starts and published
  35. * children to drain. Lifecycle pairing is host-guaranteed independently:
  36. * every forwarded `agent-start` enters a ledger, and a dead or terminated
  37. * worker's missing `agent-end` is synthesized exactly once as cancelled. On a
  38. * termination path `agentsStarted` reports the host-observed child-start count;
  39. * calls still queued worker-side for a concurrency slot are unknowable.
  40. *
  41. * @module @deepseek-ai/dsh-workflow-workerthread/host
  42. */
  43. import { Worker } from 'node:worker_threads'
  44. import type { WorkerOptions } from 'node:worker_threads'
  45. import type { Context } from 'cordis'
  46. import type { Agent } from '@deepseek-ai/dsh-agent'
  47. import { assertNever } from '@deepseek-ai/dsh-llm'
  48. import { snapshotJsonValue } from '@deepseek-ai/dsh-session'
  49. import type SubagentService from '@deepseek-ai/dsh-subagent'
  50. import type { SubagentRun } from '@deepseek-ai/dsh-subagent'
  51. import type { WorkflowAgentEndInfo, WorkflowAgentInfo, WorkflowMeta, WorkflowResult, WorkflowRun, WorkflowRunId } from '@deepseek-ai/dsh-workflow'
  52. import { renderThrown } from './realm.ts'
  53. import type { ExecutionObserver } from './runtime.ts'
  54. import { HostToWorkerType, WorkerToHostType } from './protocol.ts'
  55. import type { HostToWorkerPayloads, WorkerToHostMessage } from './protocol.ts'
  56. import type { ChildResult, ChildStartRequest, WorkerInit } from './types.ts'
  57. /** One published child and its shared quiescent-disposal transaction. */
  58. interface ChildRecord {
  59. readonly run: SubagentRun
  60. disposal?: Promise<void>
  61. }
  62. /**
  63. * Resolve the worker entry and spawn options for the current runtime shape.
  64. * Unbuilt (tsx demos, vitest — `import.meta.url` points into `src/`), the
  65. * entry is a JavaScript data-URL bootstrap. That bootstrap runs INSIDE the
  66. * user worker, registers tsx's ESM AND CommonJS transforms there, and only
  67. * then imports the TypeScript sibling. The whole mixed-module source graph
  68. * therefore receives TypeScript transformation and the tsconfig paths map in
  69. * the worker's own module-loader realm. A worker inherits no
  70. * transform pipeline from vitest (vite transforms in-process), and a parent
  71. * `--import tsx` registration is not a contract that user workers share on
  72. * every supported Node line. Built (`lib/index.js`), the entry is the sibling
  73. * bundle the package tsdown config emits and no loader is needed (`execArgv`
  74. * pinned empty in both shapes — hermetic, like the environment).
  75. *
  76. * Both shapes spawn with an EMPTY environment (`env: {}`): the documented vm
  77. * escape reaches `process`, and the harness's ambient credentials
  78. * (`DEEPSEEK_API_KEY` et al.) must not ride along — the same stance as
  79. * `dsh-code-runtime-worker`, stronger than the scrubbed env the
  80. * defensive-patterns rule requires for spawned commands (a shell needs PATH;
  81. * this worker needs nothing). Sole exception: the unbuilt shape forwards
  82. * `TSX_TSCONFIG_PATH` when the parent carries it (loader plumbing the paths
  83. * map depends on outside the repo cwd, not a secret). This closes the
  84. * AMBIENT channel only — an escapee still holds process-wide privileges
  85. * like fs access (the README's trust premise stands).
  86. * @param init - the run payload, passed as `workerData`.
  87. * @returns the entry URL and the Worker options to spawn it with.
  88. */
  89. function resolveWorkerSpawn(init: WorkerInit): { entry: URL; options: WorkerOptions } {
  90. /* v8 ignore next 3 -- the built-output arm: tests always run unbuilt (src/); the built-worker e2e exercises this shape for real */
  91. if (!import.meta.url.endsWith('.ts')) {
  92. return { entry: new URL('./worker.js', import.meta.url), options: { workerData: init, env: {}, execArgv: [] } }
  93. }
  94. // Resolve tsx lazily: only the unbuilt shape executes this arm, so a built
  95. // consumer never needs the dev-only loader installed. A JavaScript entry is
  96. // essential — it can install tsx's ESM and CommonJS hooks from INSIDE the
  97. // user worker before any TypeScript enters Node's native strip-only parser.
  98. // Both hooks are load-bearing because the source graph crosses both module
  99. // shapes on supported Node lines. TSX_TSCONFIG_PATH is
  100. // the one variable forwarded through the scrub: a parent running outside
  101. // the repo cwd (the ACP snapshot harness is the real case) pins the paths
  102. // map through it. Loader plumbing, not a secret.
  103. const workerEntry = new URL('./worker.ts', import.meta.url)
  104. const tsxEsmApiEntry = import.meta.resolve('tsx/esm/api')
  105. const tsxCjsApiEntry = import.meta.resolve('tsx/cjs/api')
  106. const bootstrap = [
  107. `import { register as registerEsm } from ${JSON.stringify(tsxEsmApiEntry)}`,
  108. `import { register as registerCjs } from ${JSON.stringify(tsxCjsApiEntry)}`,
  109. 'registerCjs()',
  110. 'registerEsm()',
  111. `await import(${JSON.stringify(workerEntry.href)})`,
  112. ].join('\n')
  113. return {
  114. entry: new URL(`data:text/javascript,${encodeURIComponent(bootstrap)}`),
  115. options: {
  116. workerData: init,
  117. env: process.env.TSX_TSCONFIG_PATH === undefined ? {} : { TSX_TSCONFIG_PATH: process.env.TSX_TSCONFIG_PATH },
  118. execArgv: [],
  119. },
  120. }
  121. }
  122. /**
  123. * One live worker-engine run — the seam's {@link WorkflowRun}, returned by
  124. * `start()` directly. Owns the Worker, the child registry, and the result
  125. * settlement; `result` never rejects. `meta` is trusted same-process data
  126. * borrowed as immutable by the handle and lifecycle events. The holder-bound
  127. * SubagentService handle is captured before the
  128. * engine returns this run, so unloading the engine removes only the ability to
  129. * start another workflow; this run can still start and clean up its children.
  130. */
  131. export class WorkerRun implements WorkflowRun {
  132. /** Settles exactly once with the run's outcome; never rejects. */
  133. readonly result: Promise<WorkflowResult>
  134. private settleResolve!: (result: WorkflowResult) => void
  135. private settled = false
  136. /** A Result/death/grace outcome atomically won before teardown callbacks. */
  137. private terminalClaimed = false
  138. /** The first death signal closes worker-message admission and owns failure-time cleanup. */
  139. private workerDeathObserved = false
  140. private cancelReason: string | undefined
  141. private graceTimer: NodeJS.Timeout | undefined
  142. private readonly worker: Worker
  143. /** Set on `exit`: the thread is gone, so posting has nowhere to go. */
  144. private workerGone = false
  145. /** Accepted `child-start` messages — the terminate-path `agentsStarted` (see module doc). */
  146. private hostStarted = 0
  147. /** Published children by callId; an entry leaves only after disposal settles. */
  148. private readonly children = new Map<number, ChildRecord>()
  149. /** Provider starts that have not yet fulfilled or rejected. */
  150. private readonly pendingStarts = new Set<Promise<void>>()
  151. /** Started-but-not-ended agents by seq — the pairing ledger the HOST guarantees (see {@link endAgent}). */
  152. private readonly liveAgents = new Map<number, WorkflowAgentInfo>()
  153. private readonly quiescenceWaiters: (() => void)[] = []
  154. /** The per-run abort fanout every child start request carries. */
  155. private readonly controller = new AbortController()
  156. /** External start signal and the exact callback installed on it, retained only until first settle/teardown. */
  157. private inputSignal: AbortSignal | undefined
  158. private inputSignalAbort: (() => void) | undefined
  159. private disposed: Promise<void> | undefined
  160. constructor(
  161. private readonly ctx: Context,
  162. private readonly subagents: SubagentService,
  163. readonly id: WorkflowRunId,
  164. readonly meta: WorkflowMeta,
  165. private readonly parent: Agent,
  166. init: WorkerInit,
  167. private readonly provider: string,
  168. private readonly disposeGraceMs: number,
  169. private readonly observer: ExecutionObserver,
  170. signal: AbortSignal | undefined,
  171. ) {
  172. this.result = new Promise<WorkflowResult>((resolve) => { this.settleResolve = resolve })
  173. // workerData rides the structured clone: args are plain JSON by the seam
  174. // contract, so the clone is total and doubles as the caller-isolation
  175. // copy (a clone failure throws loud out of start()).
  176. const { entry, options } = resolveWorkerSpawn(init)
  177. this.worker = new Worker(entry, options)
  178. this.worker.on('message', (message: WorkerToHostMessage) => { this.onMessage(message) })
  179. this.worker.on('error', (error) => { this.onWorkerDeath(`workflow worker failed: ${renderThrown(error)}`, false) })
  180. /* v8 ignore next -- messageerror: not constructible from the engine's own protocol (every payload is JSON data) */
  181. this.worker.on('messageerror', (error) => { this.onWorkerDeath(`workflow worker message failed to deserialize: ${renderThrown(error)}`, false) })
  182. this.worker.on('exit', (code) => {
  183. this.workerGone = true
  184. this.onWorkerDeath(`workflow worker exited before the run settled (exit code ${code})`, true)
  185. })
  186. if (signal?.aborted) {
  187. this.cancel('workflow start signal already aborted')
  188. } else if (signal !== undefined) {
  189. const onAbort = (): void => {
  190. this.detachInputSignal()
  191. this.cancel('workflow signal aborted')
  192. }
  193. this.inputSignal = signal
  194. this.inputSignalAbort = onAbort
  195. signal.addEventListener('abort', onAbort, { once: true })
  196. }
  197. }
  198. /**
  199. * Cancel the run: the worker is told (its hooks start throwing and the
  200. * script dies at its next await), the required signal shared by every child
  201. * start is aborted, and the grace timer
  202. * arms: a run still unsettled `disposeGraceMs` later force-settles
  203. * `cancelled` and its worker is TERMINATED. Idempotent; the first reason
  204. * wins.
  205. * @param reason - human-readable cause (default `'workflow cancelled'`).
  206. */
  207. cancel(reason?: string): void {
  208. // A settled run has nothing left to cancel, and a terminal source claimed
  209. // before its cleanup callbacks must exclude cancellation reentered by one
  210. // of those callbacks. Without the settled guard the
  211. // ordinary consumer path (await result, then dispose -> cancel) would arm
  212. // a grace timer nothing ever clears, pinning the run and its Worker
  213. // closure until the grace expires - a bounded leak per completed run.
  214. if (this.settled || this.terminalClaimed || this.cancelReason !== undefined) return
  215. this.cancelReason = reason ?? 'workflow cancelled'
  216. this.post(HostToWorkerType.Cancel, { reason: this.cancelReason })
  217. this.abortChildren(this.cancelReason)
  218. this.graceTimer = setTimeout(() => {
  219. // Cancellation already owns the race through cancelReason; close the
  220. // terminal boundary explicitly before observer teardown callbacks.
  221. this.terminalClaimed = true
  222. // The worker may no longer speak (it is about to be terminated): pair
  223. // every stranded start before the run settles, so ends precede
  224. // workflow/end.
  225. this.endStrandedAgents()
  226. this.settleResult(this.cancelledResult(this.hostStarted))
  227. void this.worker.terminate()
  228. }, this.disposeGraceMs)
  229. // unref'd: an armed grace timer must never hold the process open.
  230. this.graceTimer.unref()
  231. }
  232. /**
  233. * Cancel + bounded settle + termination. Host-drives every registered
  234. * child's disposal IMMEDIATELY — a wedged worker can relay no dispose RPC,
  235. * and deferring child teardown to the post-terminate reap would spend the
  236. * whole grace waiting for a quiescence that cannot start, then return with
  237. * the disposals still in flight — so child disposal overlaps the same
  238. * grace the worker gets to settle (the worker's own dispose RPCs join the
  239. * shared per-child disposal). Waits (at most the grace) for the result and
  240. * child quiescence, then terminates the worker unconditionally — the
  241. * thread never outlives its run — and reaps whatever children remain
  242. * (their disposal is contained, not awaited past the grace, the same
  243. * abandonment the seam documents for a slow-disposing child). Idempotent;
  244. * safe on every path.
  245. * @returns resolves when the run's resources are released or abandoned.
  246. */
  247. dispose(): Promise<void> {
  248. if (this.disposed !== undefined) return this.disposed
  249. // Claim the public transaction BEFORE its body invokes child/provider
  250. // disposal. A raw provider callback can reenter handle.dispose(); it must
  251. // join this promise rather than start a second traversal.
  252. const claimed = Promise.withResolvers<undefined>()
  253. this.disposed = claimed.promise
  254. void (async () => {
  255. this.detachInputSignal()
  256. this.cancel('workflow disposed')
  257. // cancel() deliberately becomes a no-op after terminal settlement, but
  258. // disposal still owns every registered child. Reap independently so an
  259. // already-settled workflow cannot wait on child quiescence before it has
  260. // started the surviving children's disposals. On an unsettled run this
  261. // joins the cancel path through the per-call cancellation/disposal gates.
  262. this.reapChildren('workflow disposed')
  263. await Promise.race([
  264. (async () => {
  265. await this.result
  266. await this.childQuiescence()
  267. })(),
  268. sleep(this.disposeGraceMs),
  269. ])
  270. await this.worker.terminate()
  271. this.reapChildren('workflow disposed')
  272. })().then(
  273. () => { claimed.resolve(undefined) },
  274. /* v8 ignore next -- result/quiescence never reject and Worker.terminate is the only external promise */
  275. (error: unknown) => { claimed.reject(error) },
  276. )
  277. return this.disposed
  278. }
  279. /** Post one message to the worker (payload looked up from the tag's map entry), tolerating a thread that is already gone. */
  280. private post<T extends HostToWorkerType>(type: T, payload: HostToWorkerPayloads[T]): void {
  281. if (this.workerGone || this.workerDeathObserved) return
  282. try {
  283. this.worker.postMessage({ type, ...payload })
  284. } catch (error: unknown) {
  285. // Only a teardown race can land here (every engine message is JSON
  286. // data, so serialization cannot fail); there is nothing left to
  287. // deliver to — log and move on.
  288. /* v8 ignore next -- postMessage teardown race (a throw between exit and its event): not constructible in-process */
  289. this.ctx.logger.warn(`workflow-workerthread: postMessage failed: ${renderThrown(error)}`)
  290. }
  291. }
  292. private onMessage(message: WorkerToHostMessage): void {
  293. // Node may emit `error`, then deliver an already-queued `message`, then
  294. // emit `exit`. The first death signal is the host's logical delivery
  295. // barrier: nothing arriving afterward may create a child, narrate after
  296. // workflow/end, or compete with the chosen outcome.
  297. if (this.workerDeathObserved) return
  298. switch (message.type) {
  299. case WorkerToHostType.Ready:
  300. this.post(HostToWorkerType.Go, {})
  301. break
  302. case WorkerToHostType.Phase:
  303. // Post-cancel narration is suppressed host-side: worker-side the
  304. // hooks throw once the cancel message is PROCESSED, but narration
  305. // already in flight (or emitted while the cancel crossed the
  306. // boundary) must not reach observers — nothing is emitted after
  307. // cancel() returns.
  308. if (this.cancelReason === undefined) this.observer.phase(message.title)
  309. break
  310. case WorkerToHostType.Log:
  311. if (this.cancelReason === undefined) this.observer.log(message.message)
  312. break
  313. case WorkerToHostType.AgentStart:
  314. this.liveAgents.set(message.info.seq, message.info)
  315. this.observer.agentStart(message.info)
  316. break
  317. case WorkerToHostType.AgentEnd:
  318. // NOT suppressed on cancel: cancelled children report their paired
  319. // agent-end with outcome 'cancelled'. The gate (with the termination
  320. // paths' synthesis) is what makes the one-pair-per-started-child
  321. // contract hold on every stop path.
  322. this.endAgent(message.info)
  323. break
  324. case WorkerToHostType.ChildStart:
  325. this.onChildStart(message.callId, message.request)
  326. break
  327. case WorkerToHostType.ChildDispose:
  328. this.onChildDispose(message.callId)
  329. break
  330. case WorkerToHostType.Result:
  331. this.onResult(message.result)
  332. break
  333. /* v8 ignore next 2 -- closed engine-owned union; the arm only makes adding a message type a compile error */
  334. default:
  335. assertNever(message, 'worker-to-host message')
  336. }
  337. }
  338. /** Why a ready provider result may no longer be admitted to the worker. */
  339. private childAdmissionFailure(): { reason: string; rendered: string } | undefined {
  340. if (this.cancelReason !== undefined) {
  341. return { reason: this.cancelReason, rendered: `workflow run cancelled: ${this.cancelReason}` }
  342. }
  343. if (this.workerDeathObserved) {
  344. return { reason: 'workflow worker gone', rendered: 'workflow worker is no longer available' }
  345. }
  346. if (this.terminalClaimed) {
  347. return { reason: 'workflow settled', rendered: 'workflow run already settled' }
  348. }
  349. return undefined
  350. }
  351. private onChildStart(callId: number, request: ChildStartRequest): void {
  352. const initialFailure = this.childAdmissionFailure()
  353. if (initialFailure !== undefined) {
  354. // Refuse after a terminal boundary: a child must never start on an
  355. // already-aborted signal (a provider subscribing only to future abort
  356. // events would never observe it).
  357. this.post(HostToWorkerType.ChildStartError, { callId, rendered: initialFailure.rendered })
  358. return
  359. }
  360. this.hostStarted += 1
  361. const task = this.startChild(callId, request)
  362. this.pendingStarts.add(task)
  363. void task.then(
  364. () => { this.finishPendingStart(task) },
  365. /* v8 ignore next -- startChild contains provider and cleanup failures */
  366. () => { this.finishPendingStart(task) },
  367. )
  368. }
  369. /** Await one provider-owned startup transaction and publish only while admitted. */
  370. private async startChild(callId: number, request: ChildStartRequest): Promise<void> {
  371. let run: SubagentRun
  372. try {
  373. run = await this.subagents.start(this.provider, {
  374. prompt: [{ type: 'text', text: request.prompt }],
  375. parent: this.parent,
  376. signal: this.controller.signal,
  377. ...request.schema !== undefined ? { outputSchema: request.schema } : {},
  378. ...request.model !== undefined ? { agentOptions: { model: request.model } } : {},
  379. })
  380. } catch (error: unknown) {
  381. const failure = this.childAdmissionFailure()
  382. this.post(HostToWorkerType.ChildStartError, {
  383. callId,
  384. rendered: failure?.rendered ?? renderThrown(error),
  385. })
  386. return
  387. }
  388. const failure = this.childAdmissionFailure()
  389. if (failure !== undefined) {
  390. this.post(HostToWorkerType.ChildStartError, { callId, rendered: failure.rendered })
  391. try {
  392. await run.dispose()
  393. } catch (error: unknown) {
  394. this.ctx.logger.warn(`workflow-workerthread: refused child dispose failed: ${renderThrown(error)}`)
  395. }
  396. return
  397. }
  398. const record: ChildRecord = { run }
  399. this.children.set(callId, record)
  400. // Attach result forwarding before publishing the child handle. Because the
  401. // callback itself runs in a later microtask, ChildStarted is still posted
  402. // first even for an already-settled scripted provider.
  403. const forwardResult = run.result.then<() => void, () => void>(
  404. (result) => {
  405. try {
  406. const snapshot = snapshotJsonValue<ChildResult>({
  407. output: result.output,
  408. ...result.structured !== undefined ? { structured: result.structured } : {},
  409. stopReason: result.stopReason,
  410. })
  411. if (snapshot === undefined) throw new TypeError('child result is not losslessly JSON-serializable')
  412. return () => { this.post(HostToWorkerType.ChildSettled, { callId, result: snapshot }) }
  413. } catch (error: unknown) {
  414. const rendered = `workflow child result could not cross the worker boundary: ${renderThrown(error)}`
  415. return () => { this.post(HostToWorkerType.ChildFailed, { callId, rendered }) }
  416. }
  417. },
  418. (error: unknown) => {
  419. const rendered = renderThrown(error)
  420. return () => { this.post(HostToWorkerType.ChildFailed, { callId, rendered }) }
  421. },
  422. )
  423. this.post(HostToWorkerType.ChildStarted, { callId, childId: run.id })
  424. void forwardResult.then((forward) => { forward() })
  425. }
  426. private onChildDispose(callId: number): void {
  427. const record = this.children.get(callId)
  428. if (record === undefined) {
  429. // Already disposed host-side (a dispose() drive or a death reap beat
  430. // the RPC) — the ack is still owed (the worker-side wrapper awaits it).
  431. this.post(HostToWorkerType.ChildDisposed, { callId })
  432. return
  433. }
  434. // disposeChild never rejects (containment is inside), so the ack always follows.
  435. void this.disposeChild(callId, record).then(() => { this.post(HostToWorkerType.ChildDisposed, { callId }) })
  436. }
  437. /**
  438. * Start (or join) one registered child's disposal; the registry entry
  439. * leaves when it settles. Memoized per callId: the worker's dispose RPC,
  440. * the dispose() host drive, and the reap can all land on the same child —
  441. * the child's `dispose()` runs once and every caller awaits that one
  442. * settlement. A rejection is contained (the subagent seam's dispose() is
  443. * not supposed to reject, but a backend that does anyway must not break
  444. * quiescence): logged, and the child still leaves the registry.
  445. * @param callId - the child's registry key.
  446. * @param record - the registered child (the caller looked it up).
  447. * @returns resolves when the disposal settled either way; never rejects.
  448. */
  449. private disposeChild(callId: number, record: ChildRecord): Promise<void> {
  450. if (record.disposal !== undefined) return record.disposal
  451. record.disposal = Promise.resolve()
  452. .then(() => record.run.dispose())
  453. .catch((error: unknown) => {
  454. this.ctx.logger.warn(`workflow-workerthread: child dispose failed: ${renderThrown(error)}`)
  455. })
  456. .then(() => { this.finishChild(callId, record) })
  457. return record.disposal
  458. }
  459. /** Drop an exact child record and release quiescence waiters when all work ends. */
  460. private finishChild(callId: number, record: ChildRecord): void {
  461. if (this.children.get(callId) === record) this.children.delete(callId)
  462. this.notifyChildQuiescence()
  463. }
  464. /** Retire one provider startup transaction. */
  465. private finishPendingStart(task: Promise<void>): void {
  466. this.pendingStarts.delete(task)
  467. this.notifyChildQuiescence()
  468. }
  469. /** Release waiters only after both pending starts and published children end. */
  470. private notifyChildQuiescence(): void {
  471. if (this.children.size !== 0 || this.pendingStarts.size !== 0) return
  472. for (const waiter of this.quiescenceWaiters.splice(0)) waiter()
  473. }
  474. /** Resolves once every pending start and published child has reached quiescence. */
  475. private childQuiescence(): Promise<void> {
  476. if (this.children.size === 0 && this.pendingStarts.size === 0) return Promise.resolve()
  477. return new Promise((resolve) => { this.quiescenceWaiters.push(resolve) })
  478. }
  479. /** Abort + dispose every registered child (worker death / final teardown); disposal is contained, not awaited. */
  480. private reapChildren(reason: string): void {
  481. this.abortChildren(this.cancelReason ?? reason)
  482. for (const [callId, record] of [...this.children]) {
  483. void this.disposeChild(callId, record)
  484. }
  485. }
  486. /** Abort the one canonical signal shared by pending and published children. */
  487. private abortChildren(reason: string): void {
  488. if (!this.controller.signal.aborted) this.controller.abort(reason)
  489. }
  490. private onResult(result: WorkflowResult): void {
  491. // The owned worker session sends one Result. Keep a late duplicate or a
  492. // Result queued behind another terminal source completely side-effect-free.
  493. if (this.terminalClaimed) return
  494. // First-wins is decided when the Result message reaches the host. If no
  495. // external cancellation was already in flight, this result won. Reaping a
  496. // stray child below may synchronously reenter cancel() through provider
  497. // callbacks, but that internal post-result cleanup must not retroactively
  498. // rewrite the worker result that arrived first.
  499. const cancellationWasRequested = this.cancelReason !== undefined
  500. // Claim before settlement cleanup invokes provider disposal. Once Result
  501. // won, a later cancellation cannot rewrite it.
  502. this.terminalClaimed = true
  503. // Abort pending starts and begin disposing published children before the
  504. // workflow becomes externally settled. Cleanup remains independently
  505. // tracked by childQuiescence and the holder's dispose().
  506. this.reapChildren('workflow settled')
  507. if (!cancellationWasRequested) {
  508. this.settleResult(result)
  509. return
  510. }
  511. if (result.stopReason !== 'cancelled') {
  512. // The script settled while our cancel was crossing the thread boundary
  513. // — the seam-visible result had NOT settled when cancellation was
  514. // requested, so report cancelled (the vm drive()'s post-settle check,
  515. // relocated to the receiving side of the race).
  516. this.settleResult(this.cancelledResult(result.agentsStarted))
  517. return
  518. }
  519. this.settleResult(result)
  520. }
  521. /** Process an error/messageerror/exit signal; `exit` also performs the final disposal sweep. */
  522. private onWorkerDeath(message: string, isExit: boolean): void {
  523. if (!this.workerDeathObserved) {
  524. // Close message admission BEFORE cleanup callbacks: Node can deliver a
  525. // message queued before the crash after its `error` event. Treating the
  526. // first death signal as a logical barrier prevents that late message
  527. // from creating work or narrating after workflow/end.
  528. this.workerDeathObserved = true
  529. const outcomeWasClaimed = this.terminalClaimed
  530. const cancellationWasRequested = this.cancelReason !== undefined
  531. // When death is itself the terminal source, claim BEFORE child reap or
  532. // synthesized observer callbacks. Either can reenter cancel(); a death
  533. // that arrived first remains an error, while a cancellation already
  534. // accepted before death remains cancelled. If Result/grace already won,
  535. // preserve it while still performing prompt failure-time cleanup.
  536. if (!outcomeWasClaimed) this.terminalClaimed = true
  537. if (this.children.size > 0 || this.pendingStarts.size > 0) this.reapChildren('workflow worker gone')
  538. this.endStrandedAgents()
  539. if (!outcomeWasClaimed) {
  540. if (cancellationWasRequested) {
  541. this.settleResult(this.cancelledResult(this.hostStarted))
  542. } else {
  543. this.settleResult({ value: null, stopReason: 'error', error: message, agentsStarted: this.hostStarted })
  544. }
  545. }
  546. }
  547. if (!isExit) return
  548. // `error` is not Node's physical delivery barrier: a queued message may
  549. // precede `exit`. Admission is already closed, so this final sweep only
  550. // joins/starts disposal for registry survivors; it deliberately does not
  551. // repeat explicit provider cancellation.
  552. for (const [callId, record] of [...this.children]) void this.disposeChild(callId, record)
  553. this.endStrandedAgents()
  554. }
  555. /**
  556. * The single agent-end emission gate: forwards `end` iff its start is still
  557. * unpaired in the ledger, so every forwarded `workflow/agent-start` gets
  558. * EXACTLY one `workflow/agent-end` — the worker's own report where it can
  559. * speak, a host-synthesized one where it cannot ({@link endStrandedAgents}).
  560. * @param end - the settlement to emit (worker-reported or synthesized).
  561. */
  562. private endAgent(end: WorkflowAgentEndInfo): void {
  563. /* v8 ignore next -- a real end still in flight across the grace force-settle: not orderable in-process */
  564. if (!this.liveAgents.delete(end.seq)) return
  565. this.observer.agentEnd(end)
  566. }
  567. /**
  568. * Synthesize the missing `agent-end` for every started-but-unpaired agent,
  569. * outcome `'cancelled'`: the reap cancels every child, and a real
  570. * settlement racing the force-settle loses to that already-started external
  571. * cancellation. The atomic terminal boundaries in {@link onResult} and
  572. * {@link onWorkerDeath} deliberately exclude teardown callbacks as contenders.
  573. * Called where the worker can no longer speak (the grace force-settle,
  574. * worker death, physical exit). When grace/death is the terminal source it
  575. * runs before settleResult, so already-known pairs precede `workflow/end`;
  576. * after an earlier Result, exit cleanup may close a survivor afterward.
  577. * The ledger preserves exactly-once pairing in both orders.
  578. */
  579. private endStrandedAgents(): void {
  580. for (const info of [...this.liveAgents.values()]) {
  581. this.endAgent({ ...info, outcome: 'cancelled' })
  582. }
  583. }
  584. private cancelledResult(agentsStarted: number): WorkflowResult {
  585. // cancel() is the only writer of cancelReason and every caller checks it
  586. // first; the fallback guards the type, not a reachable path.
  587. /* v8 ignore next */
  588. const reason = this.cancelReason ?? 'workflow cancelled'
  589. return { value: null, stopReason: 'cancelled', error: `workflow run cancelled: ${reason}`, agentsStarted }
  590. }
  591. /** Remove the exact abort callback installed on the caller's start signal. */
  592. private detachInputSignal(): void {
  593. const signal = this.inputSignal
  594. const onAbort = this.inputSignalAbort
  595. if (signal === undefined || onAbort === undefined) return
  596. this.inputSignal = undefined
  597. this.inputSignalAbort = undefined
  598. signal.removeEventListener('abort', onAbort)
  599. }
  600. /** First settle wins; disarms the grace timer and releases the caller signal. */
  601. private settleResult(result: WorkflowResult): void {
  602. // Every current terminal source claims ownership before calling here; keep
  603. // the fallback local so a future caller cannot resolve twice.
  604. /* v8 ignore next -- defensive fallback outside the claimed state machine */
  605. if (this.settled) return
  606. this.terminalClaimed = true
  607. this.settled = true
  608. this.detachInputSignal()
  609. clearTimeout(this.graceTimer)
  610. this.settleResolve(result)
  611. }
  612. }
  613. /** A plain timer sleep (the dispose grace); unref'd so it never holds the process open. */
  614. function sleep(ms: number): Promise<void> {
  615. return new Promise((resolve) => {
  616. const timer = setTimeout(resolve, ms)
  617. timer.unref()
  618. })
  619. }