1
0

host.ts 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594
  1. /**
  2. * Host side of one workflow run. The first worker result, unexpected death, or
  3. * cancellation-grace expiry owns settlement and closes message admission.
  4. * Pending starts share one abort signal; published children share idempotent
  5. * cleanup, and quiescence waits for both while synthesizing any missing end events.
  6. * @module @deepseek-ai/dsh-workflow-workerthread/host
  7. */
  8. import { Worker } from 'node:worker_threads'
  9. import type { WorkerOptions } from 'node:worker_threads'
  10. import { fileURLToPath } from 'node:url'
  11. import type { Context } from 'cordis'
  12. import type { Agent } from '@deepseek-ai/dsh-agent'
  13. import { assertNever } from '@deepseek-ai/dsh-llm'
  14. import { snapshotJsonValue } from '@deepseek-ai/dsh-session'
  15. import type SubagentService from '@deepseek-ai/dsh-subagent'
  16. import type { SubagentRun } from '@deepseek-ai/dsh-subagent'
  17. import type { WorkflowAgentEndInfo, WorkflowAgentInfo, WorkflowMeta, WorkflowResult, WorkflowRun, WorkflowRunId } from '@deepseek-ai/dsh-workflow'
  18. import { renderThrown } from './realm.ts'
  19. import type { ExecutionObserver } from './runtime.ts'
  20. import { HostToWorkerType, WorkerToHostType } from './protocol.ts'
  21. import type { HostToWorkerPayloads, WorkerToHostMessage } from './protocol.ts'
  22. import type { ChildResult, ChildStartRequest, WorkerInit } from './types.ts'
  23. /** One published child and its shared quiescent-disposal transaction. */
  24. interface ChildRecord {
  25. readonly run: SubagentRun
  26. disposal?: Promise<void>
  27. }
  28. /**
  29. * Resolve a built worker bundle or an unbuilt bootstrap that installs both tsx
  30. * transforms inside the worker. Both shapes clear `execArgv` and the ambient
  31. * environment; the unbuilt shape forwards only `TSX_TSCONFIG_PATH` for path
  32. * resolution.
  33. * @param init - the run payload, passed as `workerData`.
  34. * @returns the entry path or URL and the Worker options to spawn it with.
  35. */
  36. function resolveWorkerSpawn(init: WorkerInit): { entry: string | URL; options: WorkerOptions } {
  37. /* v8 ignore next 3 -- the built-output arm: tests always run unbuilt (src/); the built-worker e2e exercises this shape for real */
  38. if (!import.meta.url.endsWith('.ts')) {
  39. return { entry: fileURLToPath(new URL('./worker.cjs', import.meta.url)), options: { workerData: init, env: {}, execArgv: [] } }
  40. }
  41. // Resolve tsx only for unbuilt consumers and install it before importing TS.
  42. const workerEntry = new URL('./worker.ts', import.meta.url)
  43. const tsxEsmApiEntry = import.meta.resolve('tsx/esm/api')
  44. const tsxCjsApiEntry = import.meta.resolve('tsx/cjs/api')
  45. const bootstrap = [
  46. `import { register as registerEsm } from ${JSON.stringify(tsxEsmApiEntry)}`,
  47. `import { register as registerCjs } from ${JSON.stringify(tsxCjsApiEntry)}`,
  48. 'registerCjs()',
  49. 'registerEsm()',
  50. `await import(${JSON.stringify(workerEntry.href)})`,
  51. ].join('\n')
  52. return {
  53. entry: new URL(`data:text/javascript,${encodeURIComponent(bootstrap)}`),
  54. options: {
  55. workerData: init,
  56. env: process.env.TSX_TSCONFIG_PATH === undefined ? {} : { TSX_TSCONFIG_PATH: process.env.TSX_TSCONFIG_PATH },
  57. execArgv: [],
  58. },
  59. }
  60. }
  61. /**
  62. * One live worker-engine run — the seam's {@link WorkflowRun}, returned by
  63. * `start()` directly. Owns the Worker, the child registry, and the result
  64. * settlement; `result` never rejects. `meta` is trusted same-process data
  65. * borrowed as immutable by the handle and lifecycle events. The holder-bound
  66. * SubagentService handle is captured before the
  67. * engine returns this run, so unloading the engine removes only the ability to
  68. * start another workflow; this run can still start and clean up its children.
  69. */
  70. export class WorkerRun implements WorkflowRun {
  71. /** Settles exactly once with the run's outcome; never rejects. */
  72. readonly result: Promise<WorkflowResult>
  73. private settleResolve!: (result: WorkflowResult) => void
  74. private settled = false
  75. /** A Result/death/grace outcome atomically won before teardown callbacks. */
  76. private terminalClaimed = false
  77. /** The first death signal closes worker-message admission and owns failure-time cleanup. */
  78. private workerDeathObserved = false
  79. private cancelReason: string | undefined
  80. private graceTimer: NodeJS.Timeout | undefined
  81. private readonly worker: Worker
  82. /** Set on `exit`: the thread is gone, so posting has nowhere to go. */
  83. private workerGone = false
  84. /** Accepted `child-start` messages — the terminate-path `agentsStarted` (see module doc). */
  85. private hostStarted = 0
  86. /** Published children by callId; an entry leaves only after disposal settles. */
  87. private readonly children = new Map<number, ChildRecord>()
  88. /** Provider starts that have not yet fulfilled or rejected. */
  89. private readonly pendingStarts = new Set<Promise<void>>()
  90. /** Started-but-not-ended agents by seq — the pairing ledger the HOST guarantees (see {@link endAgent}). */
  91. private readonly liveAgents = new Map<number, WorkflowAgentInfo>()
  92. private readonly quiescenceWaiters: (() => void)[] = []
  93. /** The per-run abort fanout every child start request carries. */
  94. private readonly controller = new AbortController()
  95. /** External start signal and the exact callback installed on it, retained only until first settle/teardown. */
  96. private inputSignal: AbortSignal | undefined
  97. private inputSignalAbort: (() => void) | undefined
  98. private disposed: Promise<void> | undefined
  99. constructor(
  100. private readonly ctx: Context,
  101. private readonly subagents: SubagentService,
  102. readonly id: WorkflowRunId,
  103. readonly meta: WorkflowMeta,
  104. private readonly parent: Agent,
  105. init: WorkerInit,
  106. private readonly provider: string,
  107. private readonly disposeGraceMs: number,
  108. private readonly observer: ExecutionObserver,
  109. signal: AbortSignal | undefined,
  110. ) {
  111. this.result = new Promise<WorkflowResult>((resolve) => { this.settleResolve = resolve })
  112. // workerData rides the structured clone: args are plain JSON by the seam
  113. // contract, so the clone is total and doubles as the caller-isolation
  114. // copy (a clone failure throws loud out of start()).
  115. const { entry, options } = resolveWorkerSpawn(init)
  116. this.worker = new Worker(entry, options)
  117. this.worker.on('message', (message: WorkerToHostMessage) => { this.onMessage(message) })
  118. this.worker.on('error', (error) => { this.onWorkerDeath(`workflow worker failed: ${renderThrown(error)}`, false) })
  119. /* v8 ignore next -- messageerror: not constructible from the engine's own protocol (every payload is JSON data) */
  120. this.worker.on('messageerror', (error) => { this.onWorkerDeath(`workflow worker message failed to deserialize: ${renderThrown(error)}`, false) })
  121. this.worker.on('exit', (code) => {
  122. this.workerGone = true
  123. this.onWorkerDeath(`workflow worker exited before the run settled (exit code ${code})`, true)
  124. })
  125. if (signal?.aborted) {
  126. this.cancel('workflow start signal already aborted')
  127. } else if (signal !== undefined) {
  128. const onAbort = (): void => {
  129. this.detachInputSignal()
  130. this.cancel('workflow signal aborted')
  131. }
  132. this.inputSignal = signal
  133. this.inputSignalAbort = onAbort
  134. signal.addEventListener('abort', onAbort, { once: true })
  135. }
  136. }
  137. /**
  138. * Cancel the run: the worker is told (its hooks start throwing and the
  139. * script dies at its next await), the required signal shared by every child
  140. * start is aborted, and the grace timer
  141. * arms: a run still unsettled `disposeGraceMs` later force-settles
  142. * `cancelled` and its worker is TERMINATED. Idempotent; the first reason
  143. * wins.
  144. * @param reason - human-readable cause (default `'workflow cancelled'`).
  145. */
  146. cancel(reason?: string): void {
  147. // A settled run has nothing left to cancel, and a terminal source claimed
  148. // before its cleanup callbacks must exclude cancellation reentered by one
  149. // of those callbacks. Without the settled guard the
  150. // ordinary consumer path (await result, then dispose -> cancel) would arm
  151. // a grace timer nothing ever clears, pinning the run and its Worker
  152. // closure until the grace expires - a bounded leak per completed run.
  153. if (this.settled || this.terminalClaimed || this.cancelReason !== undefined) return
  154. this.cancelReason = reason ?? 'workflow cancelled'
  155. this.post(HostToWorkerType.Cancel, { reason: this.cancelReason })
  156. this.abortChildren(this.cancelReason)
  157. this.graceTimer = setTimeout(() => {
  158. // Cancellation already owns the race through cancelReason; close the
  159. // terminal boundary explicitly before observer teardown callbacks.
  160. this.terminalClaimed = true
  161. // The worker may no longer speak (it is about to be terminated): pair
  162. // every stranded start before the run settles, so ends precede
  163. // workflow/end.
  164. this.endStrandedAgents()
  165. this.settleResult(this.cancelledResult(this.hostStarted))
  166. void this.worker.terminate()
  167. }, this.disposeGraceMs)
  168. // unref'd: an armed grace timer must never hold the process open.
  169. this.graceTimer.unref()
  170. }
  171. /**
  172. * Cancel + bounded settle + termination. Host-drives every registered
  173. * child's disposal IMMEDIATELY — a wedged worker can relay no dispose RPC,
  174. * and deferring child teardown to the post-terminate reap would spend the
  175. * whole grace waiting for a quiescence that cannot start, then return with
  176. * the disposals still in flight — so child disposal overlaps the same
  177. * grace the worker gets to settle (the worker's own dispose RPCs join the
  178. * shared per-child disposal). Waits (at most the grace) for the result and
  179. * child quiescence, then terminates the worker unconditionally — the
  180. * thread never outlives its run — and reaps whatever children remain
  181. * (their disposal is contained, not awaited past the grace, the same
  182. * abandonment the seam documents for a slow-disposing child). Idempotent;
  183. * safe on every path.
  184. * @returns resolves when the run's resources are released or abandoned.
  185. */
  186. dispose(): Promise<void> {
  187. if (this.disposed !== undefined) return this.disposed
  188. // Claim the public transaction BEFORE its body invokes child/provider
  189. // disposal. A raw provider callback can reenter handle.dispose(); it must
  190. // join this promise rather than start a second traversal.
  191. const claimed = Promise.withResolvers<undefined>()
  192. this.disposed = claimed.promise
  193. void (async () => {
  194. this.detachInputSignal()
  195. this.cancel('workflow disposed')
  196. // cancel() deliberately becomes a no-op after terminal settlement, but
  197. // disposal still owns every registered child. Reap independently so an
  198. // already-settled workflow cannot wait on child quiescence before it has
  199. // started the surviving children's disposals. On an unsettled run this
  200. // joins the cancel path through the per-call cancellation/disposal gates.
  201. this.reapChildren('workflow disposed')
  202. await Promise.race([
  203. (async () => {
  204. await this.result
  205. await this.childQuiescence()
  206. })(),
  207. sleep(this.disposeGraceMs),
  208. ])
  209. await this.worker.terminate()
  210. this.reapChildren('workflow disposed')
  211. })().then(
  212. () => { claimed.resolve(undefined) },
  213. /* v8 ignore next -- result/quiescence never reject and Worker.terminate is the only external promise */
  214. (error: unknown) => { claimed.reject(error) },
  215. )
  216. return this.disposed
  217. }
  218. /** Post one message to the worker (payload looked up from the tag's map entry), tolerating a thread that is already gone. */
  219. private post<T extends HostToWorkerType>(type: T, payload: HostToWorkerPayloads[T]): void {
  220. if (this.workerGone || this.workerDeathObserved) return
  221. try {
  222. this.worker.postMessage({ type, ...payload })
  223. } catch (error: unknown) {
  224. // Only a teardown race can land here (every engine message is JSON
  225. // data, so serialization cannot fail); there is nothing left to
  226. // deliver to — log and move on.
  227. /* v8 ignore next -- postMessage teardown race (a throw between exit and its event): not constructible in-process */
  228. this.ctx.logger.warn(`workflow-workerthread: postMessage failed: ${renderThrown(error)}`)
  229. }
  230. }
  231. private onMessage(message: WorkerToHostMessage): void {
  232. // Node may emit `error`, then deliver an already-queued `message`, then
  233. // emit `exit`. The first death signal is the host's logical delivery
  234. // barrier: nothing arriving afterward may create a child, narrate after
  235. // workflow/end, or compete with the chosen outcome.
  236. if (this.workerDeathObserved) return
  237. switch (message.type) {
  238. case WorkerToHostType.Ready:
  239. this.post(HostToWorkerType.Go, {})
  240. break
  241. case WorkerToHostType.Phase:
  242. // Post-cancel narration is suppressed host-side: worker-side the
  243. // hooks throw once the cancel message is PROCESSED, but narration
  244. // already in flight (or emitted while the cancel crossed the
  245. // boundary) must not reach observers — nothing is emitted after
  246. // cancel() returns.
  247. if (this.cancelReason === undefined) this.observer.phase(message.title)
  248. break
  249. case WorkerToHostType.Log:
  250. if (this.cancelReason === undefined) this.observer.log(message.message)
  251. break
  252. case WorkerToHostType.AgentStart:
  253. this.liveAgents.set(message.info.seq, message.info)
  254. this.observer.agentStart(message.info)
  255. break
  256. case WorkerToHostType.AgentEnd:
  257. // NOT suppressed on cancel: cancelled children report their paired
  258. // agent-end with outcome 'cancelled'. The gate (with the termination
  259. // paths' synthesis) is what makes the one-pair-per-started-child
  260. // contract hold on every stop path.
  261. this.endAgent(message.info)
  262. break
  263. case WorkerToHostType.ChildStart:
  264. this.onChildStart(message.callId, message.request)
  265. break
  266. case WorkerToHostType.ChildDispose:
  267. this.onChildDispose(message.callId)
  268. break
  269. case WorkerToHostType.Result:
  270. this.onResult(message.result)
  271. break
  272. /* v8 ignore next 2 -- closed engine-owned union; the arm only makes adding a message type a compile error */
  273. default:
  274. assertNever(message, 'worker-to-host message')
  275. }
  276. }
  277. /** Why a ready provider result may no longer be admitted to the worker. */
  278. private childAdmissionFailure(): { reason: string; rendered: string } | undefined {
  279. if (this.cancelReason !== undefined) {
  280. return { reason: this.cancelReason, rendered: `workflow run cancelled: ${this.cancelReason}` }
  281. }
  282. if (this.workerDeathObserved) {
  283. return { reason: 'workflow worker gone', rendered: 'workflow worker is no longer available' }
  284. }
  285. if (this.terminalClaimed) {
  286. return { reason: 'workflow settled', rendered: 'workflow run already settled' }
  287. }
  288. return undefined
  289. }
  290. private onChildStart(callId: number, request: ChildStartRequest): void {
  291. const initialFailure = this.childAdmissionFailure()
  292. if (initialFailure !== undefined) {
  293. // Refuse after a terminal boundary: a child must never start on an
  294. // already-aborted signal (a provider subscribing only to future abort
  295. // events would never observe it).
  296. this.post(HostToWorkerType.ChildStartError, { callId, rendered: initialFailure.rendered })
  297. return
  298. }
  299. this.hostStarted += 1
  300. const task = this.startChild(callId, request)
  301. this.pendingStarts.add(task)
  302. void task.then(
  303. () => { this.finishPendingStart(task) },
  304. /* v8 ignore next -- startChild contains provider and cleanup failures */
  305. () => { this.finishPendingStart(task) },
  306. )
  307. }
  308. /** Await one provider-owned startup transaction and publish only while admitted. */
  309. private async startChild(callId: number, request: ChildStartRequest): Promise<void> {
  310. let run: SubagentRun
  311. try {
  312. run = await this.subagents.start(this.provider, {
  313. prompt: [{ type: 'text', text: request.prompt }],
  314. parent: this.parent,
  315. signal: this.controller.signal,
  316. ...request.schema !== undefined ? { outputSchema: request.schema } : {},
  317. ...request.provider !== undefined || request.model !== undefined
  318. ? {
  319. agentOptions: {
  320. ...request.provider !== undefined ? { provider: request.provider } : {},
  321. ...request.model !== undefined ? { model: request.model } : {},
  322. },
  323. }
  324. : {},
  325. })
  326. } catch (error: unknown) {
  327. const failure = this.childAdmissionFailure()
  328. this.post(HostToWorkerType.ChildStartError, {
  329. callId,
  330. rendered: failure?.rendered ?? renderThrown(error),
  331. })
  332. return
  333. }
  334. const failure = this.childAdmissionFailure()
  335. if (failure !== undefined) {
  336. this.post(HostToWorkerType.ChildStartError, { callId, rendered: failure.rendered })
  337. try {
  338. await run.dispose()
  339. } catch (error: unknown) {
  340. this.ctx.logger.warn(`workflow-workerthread: refused child dispose failed: ${renderThrown(error)}`)
  341. }
  342. return
  343. }
  344. const record: ChildRecord = { run }
  345. this.children.set(callId, record)
  346. // Attach result forwarding before publishing the child handle. Because the
  347. // callback itself runs in a later microtask, ChildStarted is still posted
  348. // first even for an already-settled scripted provider.
  349. const forwardResult = run.result.then<() => void, () => void>(
  350. (result) => {
  351. try {
  352. const snapshot = snapshotJsonValue<ChildResult>({
  353. output: result.output,
  354. ...result.structured !== undefined ? { structured: result.structured } : {},
  355. stopReason: result.stopReason,
  356. })
  357. if (snapshot === undefined) throw new TypeError('child result is not losslessly JSON-serializable')
  358. return () => { this.post(HostToWorkerType.ChildSettled, { callId, result: snapshot }) }
  359. } catch (error: unknown) {
  360. const rendered = `workflow child result could not cross the worker boundary: ${renderThrown(error)}`
  361. return () => { this.post(HostToWorkerType.ChildFailed, { callId, rendered }) }
  362. }
  363. },
  364. (error: unknown) => {
  365. const rendered = renderThrown(error)
  366. return () => { this.post(HostToWorkerType.ChildFailed, { callId, rendered }) }
  367. },
  368. )
  369. this.post(HostToWorkerType.ChildStarted, { callId, childId: run.id })
  370. void forwardResult.then((forward) => { forward() })
  371. }
  372. private onChildDispose(callId: number): void {
  373. const record = this.children.get(callId)
  374. if (record === undefined) {
  375. // Already disposed host-side (a dispose() drive or a death reap beat
  376. // the RPC) — the ack is still owed (the worker-side wrapper awaits it).
  377. this.post(HostToWorkerType.ChildDisposed, { callId })
  378. return
  379. }
  380. // disposeChild never rejects (containment is inside), so the ack always follows.
  381. void this.disposeChild(callId, record).then(() => { this.post(HostToWorkerType.ChildDisposed, { callId }) })
  382. }
  383. /**
  384. * Start (or join) one registered child's disposal; the registry entry
  385. * leaves when it settles. Memoized per callId: the worker's dispose RPC,
  386. * the dispose() host drive, and the reap can all land on the same child —
  387. * the child's `dispose()` runs once and every caller awaits that one
  388. * settlement. A rejection is contained (the subagent seam's dispose() is
  389. * not supposed to reject, but a backend that does anyway must not break
  390. * quiescence): logged, and the child still leaves the registry.
  391. * @param callId - the child's registry key.
  392. * @param record - the registered child (the caller looked it up).
  393. * @returns resolves when the disposal settled either way; never rejects.
  394. */
  395. private disposeChild(callId: number, record: ChildRecord): Promise<void> {
  396. if (record.disposal !== undefined) return record.disposal
  397. record.disposal = Promise.resolve()
  398. .then(() => record.run.dispose())
  399. .catch((error: unknown) => {
  400. this.ctx.logger.warn(`workflow-workerthread: child dispose failed: ${renderThrown(error)}`)
  401. })
  402. .then(() => { this.finishChild(callId) })
  403. return record.disposal
  404. }
  405. /** Drop a child record and release quiescence waiters when all work ends. */
  406. private finishChild(callId: number): void {
  407. this.children.delete(callId)
  408. this.notifyChildQuiescence()
  409. }
  410. /** Retire one provider startup transaction. */
  411. private finishPendingStart(task: Promise<void>): void {
  412. this.pendingStarts.delete(task)
  413. this.notifyChildQuiescence()
  414. }
  415. /** Release waiters only after both pending starts and published children end. */
  416. private notifyChildQuiescence(): void {
  417. if (this.children.size !== 0 || this.pendingStarts.size !== 0) return
  418. for (const waiter of this.quiescenceWaiters.splice(0)) waiter()
  419. }
  420. /** Resolves once every pending start and published child has reached quiescence. */
  421. private childQuiescence(): Promise<void> {
  422. if (this.children.size === 0 && this.pendingStarts.size === 0) return Promise.resolve()
  423. return new Promise((resolve) => { this.quiescenceWaiters.push(resolve) })
  424. }
  425. /** Abort + dispose every registered child (worker death / final teardown); disposal is contained, not awaited. */
  426. private reapChildren(reason: string): void {
  427. this.abortChildren(this.cancelReason ?? reason)
  428. for (const [callId, record] of [...this.children]) {
  429. void this.disposeChild(callId, record)
  430. }
  431. }
  432. /** Abort the one canonical signal shared by pending and published children. */
  433. private abortChildren(reason: string): void {
  434. if (!this.controller.signal.aborted) this.controller.abort(reason)
  435. }
  436. private onResult(result: WorkflowResult): void {
  437. // The owned worker session sends one Result. Keep a late duplicate or a
  438. // Result queued behind another terminal source completely side-effect-free.
  439. if (this.terminalClaimed) return
  440. // First-wins is decided when the Result message reaches the host. If no
  441. // external cancellation was already in flight, this result won. Reaping a
  442. // stray child below may synchronously reenter cancel() through provider
  443. // callbacks, but that internal post-result cleanup must not retroactively
  444. // rewrite the worker result that arrived first.
  445. const cancellationWasRequested = this.cancelReason !== undefined
  446. // Claim before settlement cleanup invokes provider disposal. Once Result
  447. // won, a later cancellation cannot rewrite it.
  448. this.terminalClaimed = true
  449. // Abort pending starts and begin disposing published children before the
  450. // workflow becomes externally settled. Cleanup remains independently
  451. // tracked by childQuiescence and the holder's dispose().
  452. this.reapChildren('workflow settled')
  453. if (!cancellationWasRequested) {
  454. this.settleResult(result)
  455. return
  456. }
  457. if (result.stopReason !== 'cancelled') {
  458. // The script settled while our cancel was crossing the thread boundary
  459. // — the seam-visible result had NOT settled when cancellation was
  460. // requested, so report cancelled (the vm drive()'s post-settle check,
  461. // relocated to the receiving side of the race).
  462. this.settleResult(this.cancelledResult(result.agentsStarted))
  463. return
  464. }
  465. this.settleResult(result)
  466. }
  467. /** Process an error/messageerror/exit signal; `exit` also performs the final disposal sweep. */
  468. private onWorkerDeath(message: string, isExit: boolean): void {
  469. if (!this.workerDeathObserved) {
  470. // Close message admission BEFORE cleanup callbacks: Node can deliver a
  471. // message queued before the crash after its `error` event. Treating the
  472. // first death signal as a logical barrier prevents that late message
  473. // from creating work or narrating after workflow/end.
  474. this.workerDeathObserved = true
  475. const outcomeWasClaimed = this.terminalClaimed
  476. const cancellationWasRequested = this.cancelReason !== undefined
  477. // When death is itself the terminal source, claim BEFORE child reap or
  478. // synthesized observer callbacks. Either can reenter cancel(); a death
  479. // that arrived first remains an error, while a cancellation already
  480. // accepted before death remains cancelled. If Result/grace already won,
  481. // preserve it while still performing prompt failure-time cleanup.
  482. if (!outcomeWasClaimed) this.terminalClaimed = true
  483. if (this.children.size > 0 || this.pendingStarts.size > 0) this.reapChildren('workflow worker gone')
  484. this.endStrandedAgents()
  485. if (!outcomeWasClaimed) {
  486. if (cancellationWasRequested) {
  487. this.settleResult(this.cancelledResult(this.hostStarted))
  488. } else {
  489. this.settleResult({ value: null, stopReason: 'error', error: message, agentsStarted: this.hostStarted })
  490. }
  491. }
  492. }
  493. if (!isExit) return
  494. // `error` is not Node's physical delivery barrier: a queued message may
  495. // precede `exit`. Admission is already closed, so this final sweep only
  496. // joins/starts disposal for registry survivors; it deliberately does not
  497. // repeat explicit provider cancellation.
  498. for (const [callId, record] of [...this.children]) void this.disposeChild(callId, record)
  499. this.endStrandedAgents()
  500. }
  501. /**
  502. * The single agent-end emission gate: forwards `end` iff its start is still
  503. * unpaired in the ledger, so every forwarded `workflow/agent-start` gets
  504. * EXACTLY one `workflow/agent-end` — the worker's own report where it can
  505. * speak, a host-synthesized one where it cannot ({@link endStrandedAgents}).
  506. * @param end - the settlement to emit (worker-reported or synthesized).
  507. */
  508. private endAgent(end: WorkflowAgentEndInfo): void {
  509. /* v8 ignore next -- a real end still in flight across the grace force-settle: not orderable in-process */
  510. if (!this.liveAgents.delete(end.seq)) return
  511. this.observer.agentEnd(end)
  512. }
  513. /**
  514. * Synthesize the missing `agent-end` for every started-but-unpaired agent,
  515. * outcome `'cancelled'`: the reap cancels every child, and a real
  516. * settlement racing the force-settle loses to that already-started external
  517. * cancellation. The atomic terminal boundaries in {@link onResult} and
  518. * {@link onWorkerDeath} deliberately exclude teardown callbacks as contenders.
  519. * Called where the worker can no longer speak (the grace force-settle,
  520. * worker death, physical exit). When grace/death is the terminal source it
  521. * runs before settleResult, so already-known pairs precede `workflow/end`;
  522. * after an earlier Result, exit cleanup may close a survivor afterward.
  523. * The ledger preserves exactly-once pairing in both orders.
  524. */
  525. private endStrandedAgents(): void {
  526. for (const info of [...this.liveAgents.values()]) {
  527. this.endAgent({ ...info, outcome: 'cancelled' })
  528. }
  529. }
  530. private cancelledResult(agentsStarted: number): WorkflowResult {
  531. // cancel() is the only writer of cancelReason and every caller checks it
  532. // first; the fallback guards the type, not a reachable path.
  533. /* v8 ignore next */
  534. const reason = this.cancelReason ?? 'workflow cancelled'
  535. return { value: null, stopReason: 'cancelled', error: `workflow run cancelled: ${reason}`, agentsStarted }
  536. }
  537. /** Remove the exact abort callback installed on the caller's start signal. */
  538. private detachInputSignal(): void {
  539. const signal = this.inputSignal
  540. const onAbort = this.inputSignalAbort
  541. if (signal === undefined || onAbort === undefined) return
  542. this.inputSignal = undefined
  543. this.inputSignalAbort = undefined
  544. signal.removeEventListener('abort', onAbort)
  545. }
  546. /** First settle wins; disarms the grace timer and releases the caller signal. */
  547. private settleResult(result: WorkflowResult): void {
  548. // Every current terminal source claims ownership before calling here; keep
  549. // the fallback local so a future caller cannot resolve twice.
  550. /* v8 ignore next -- defensive fallback outside the claimed state machine */
  551. if (this.settled) return
  552. this.terminalClaimed = true
  553. this.settled = true
  554. this.detachInputSignal()
  555. clearTimeout(this.graceTimer)
  556. this.settleResolve(result)
  557. }
  558. }
  559. /** A plain timer sleep (the dispose grace); unref'd so it never holds the process open. */
  560. function sleep(ms: number): Promise<void> {
  561. return new Promise((resolve) => {
  562. const timer = setTimeout(resolve, ms)
  563. timer.unref()
  564. })
  565. }