host.ts 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622
  1. /**
  2. * Host side of one workflow run. The first worker result, unexpected death, or
  3. * cancellation-grace expiry owns settlement and closes message admission.
  4. * Pending starts share one abort signal; published children share idempotent
  5. * cleanup, and quiescence waits for both while synthesizing any missing end events.
  6. * @module @deepseek-ai/dsh-workflow-worker-thread/host
  7. */
  8. import { tmpdir } from 'node:os'
  9. import { Worker } from 'node:worker_threads'
  10. import type { WorkerOptions } from 'node:worker_threads'
  11. import { fileURLToPath } from 'node:url'
  12. import type { Context } from '@deepseek-ai/cordis'
  13. import type { Agent } from '@deepseek-ai/dsh-agent'
  14. import { assertNever } from '@deepseek-ai/dsh-llm'
  15. import { snapshotJsonValue } from '@deepseek-ai/dsh-session'
  16. import type SubagentRuntime from '@deepseek-ai/dsh-subagent'
  17. import type { SubagentRun } from '@deepseek-ai/dsh-subagent'
  18. import type { WorkflowAgentEndInfo, WorkflowAgentInfo, WorkflowMeta, WorkflowResult, WorkflowRun, WorkflowRunId } from '@deepseek-ai/dsh-workflow'
  19. import { renderThrown } from './realm.ts'
  20. import type { ExecutionObserver } from './runtime.ts'
  21. import { HostToWorkerType, WorkerToHostType } from './protocol.ts'
  22. import type { HostToWorkerPayloads, WorkerToHostMessage } from './protocol.ts'
  23. import type { ChildResult, ChildStartRequest, WorkerInit } from './types.ts'
  24. /** One published child and its shared quiescent-disposal transaction. */
  25. interface ChildRecord {
  26. readonly run: SubagentRun
  27. disposal?: Promise<void>
  28. }
  29. /**
  30. * The scrubbed worker environment: no ambient credentials, no loader flags.
  31. * Windows derives `os.tmpdir()` from `TMP`/`TEMP` and falls back to the
  32. * literal relative path `undefined\temp` when the environment is empty, so
  33. * tsx's transform cache would land in a cwd-relative `undefined/temp`
  34. * directory; the host's real temp path (not a credential) is injected there.
  35. * The unbuilt shape additionally forwards `TSX_TSCONFIG_PATH` for path
  36. * resolution.
  37. * @param platform - host platform; overridable so tests exercise both peer arms.
  38. * @param tsconfigPath - the tsconfig pin to forward; only the unbuilt caller
  39. * passes one, so the built worker never observes the host's pin.
  40. * @returns the scrubbed worker environment object.
  41. */
  42. export function workerSpawnEnv(
  43. platform: NodeJS.Platform = process.platform,
  44. tsconfigPath?: string,
  45. ): NodeJS.ProcessEnv {
  46. const env: NodeJS.ProcessEnv = {}
  47. if (platform === 'win32') {
  48. const tmp = tmpdir()
  49. env.TMP = tmp
  50. env.TEMP = tmp
  51. }
  52. if (tsconfigPath !== undefined) env.TSX_TSCONFIG_PATH = tsconfigPath
  53. return env
  54. }
  55. /**
  56. * Resolve a built worker bundle or an unbuilt bootstrap that installs both tsx
  57. * transforms inside the worker. Both shapes clear `execArgv` and the ambient
  58. * environment (the worker only sees the platform temp path and, unbuilt,
  59. * `TSX_TSCONFIG_PATH`).
  60. * @param init - the run payload, passed as `workerData`.
  61. * @returns the entry path or URL and the Worker options to spawn it with.
  62. */
  63. function resolveWorkerSpawn(init: WorkerInit): { entry: string | URL; options: WorkerOptions } {
  64. /* v8 ignore next 3 -- the built-output arm: tests always run unbuilt (src/); the built-worker e2e exercises this shape for real */
  65. if (!import.meta.url.endsWith('.ts')) {
  66. return { entry: fileURLToPath(new URL('./worker.cjs', import.meta.url)), options: { workerData: init, env: workerSpawnEnv(), execArgv: [] } }
  67. }
  68. // Resolve tsx only for unbuilt consumers and install it before importing TS.
  69. const workerEntry = new URL('./worker.ts', import.meta.url)
  70. const tsxEsmApiEntry = import.meta.resolve('tsx/esm/api')
  71. const tsxCjsApiEntry = import.meta.resolve('tsx/cjs/api')
  72. const bootstrap = [
  73. `import { register as registerEsm } from ${JSON.stringify(tsxEsmApiEntry)}`,
  74. `import { register as registerCjs } from ${JSON.stringify(tsxCjsApiEntry)}`,
  75. 'registerCjs()',
  76. 'registerEsm()',
  77. `await import(${JSON.stringify(workerEntry.href)})`,
  78. ].join('\n')
  79. return {
  80. entry: new URL(`data:text/javascript,${encodeURIComponent(bootstrap)}`),
  81. options: {
  82. workerData: init,
  83. env: workerSpawnEnv(undefined, process.env.TSX_TSCONFIG_PATH),
  84. execArgv: [],
  85. },
  86. }
  87. }
  88. /**
  89. * One live worker-engine run — the seam's {@link WorkflowRun}, returned by
  90. * `start()` directly. Owns the Worker, the child registry, and the result
  91. * settlement; `result` never rejects. `meta` is trusted same-process data
  92. * borrowed as immutable by the handle and lifecycle events. The holder-bound
  93. * SubagentRuntime handle is captured before the
  94. * engine returns this run, so unloading the engine removes only the ability to
  95. * start another workflow; this run can still start and clean up its children.
  96. */
  97. export class WorkerRun implements WorkflowRun {
  98. /** Settles exactly once with the run's outcome; never rejects. */
  99. readonly result: Promise<WorkflowResult>
  100. private settleResolve!: (result: WorkflowResult) => void
  101. private settled = false
  102. /** A Result/death/grace outcome atomically won before teardown callbacks. */
  103. private terminalClaimed = false
  104. /** The first death signal closes worker-message admission and owns failure-time cleanup. */
  105. private workerDeathObserved = false
  106. private cancelReason: string | undefined
  107. private graceTimer: NodeJS.Timeout | undefined
  108. private readonly worker: Worker
  109. /** Set on `exit`: the thread is gone, so posting has nowhere to go. */
  110. private workerGone = false
  111. /** Accepted `child-start` messages — the terminate-path `agentsStarted` (see module doc). */
  112. private hostStarted = 0
  113. /** Published children by callId; an entry leaves only after disposal settles. */
  114. private readonly children = new Map<number, ChildRecord>()
  115. /** Provider starts that have not yet fulfilled or rejected. */
  116. private readonly pendingStarts = new Set<Promise<void>>()
  117. /** Started-but-not-ended agents by seq — the pairing ledger the HOST guarantees (see {@link endAgent}). */
  118. private readonly liveAgents = new Map<number, WorkflowAgentInfo>()
  119. private readonly quiescenceWaiters: (() => void)[] = []
  120. /** The per-run abort fanout every child start request carries. */
  121. private readonly controller = new AbortController()
  122. /** External start signal and the exact callback installed on it, retained only until first settle/teardown. */
  123. private inputSignal: AbortSignal | undefined
  124. private inputSignalAbort: (() => void) | undefined
  125. private disposed: Promise<void> | undefined
  126. constructor(
  127. private readonly ctx: Context,
  128. private readonly subagents: SubagentRuntime,
  129. readonly id: WorkflowRunId,
  130. readonly meta: WorkflowMeta,
  131. private readonly parent: Agent,
  132. init: WorkerInit,
  133. private readonly provider: string,
  134. private readonly disposeGraceMs: number,
  135. private readonly observer: ExecutionObserver,
  136. signal: AbortSignal | undefined,
  137. ) {
  138. this.result = new Promise<WorkflowResult>((resolve) => { this.settleResolve = resolve })
  139. // workerData rides the structured clone: args are plain JSON by the seam
  140. // contract, so the clone is total and doubles as the caller-isolation
  141. // copy (a clone failure throws loud out of start()).
  142. const { entry, options } = resolveWorkerSpawn(init)
  143. this.worker = new Worker(entry, options)
  144. this.worker.on('message', (message: WorkerToHostMessage) => { this.onMessage(message) })
  145. this.worker.on('error', (error) => { this.onWorkerDeath(`workflow worker failed: ${renderThrown(error)}`, false) })
  146. /* v8 ignore next -- messageerror: not constructible from the engine's own protocol (every payload is JSON data) */
  147. this.worker.on('messageerror', (error) => { this.onWorkerDeath(`workflow worker message failed to deserialize: ${renderThrown(error)}`, false) })
  148. this.worker.on('exit', (code) => {
  149. this.workerGone = true
  150. this.onWorkerDeath(`workflow worker exited before the run settled (exit code ${code})`, true)
  151. })
  152. if (signal?.aborted) {
  153. this.cancel('workflow start signal already aborted')
  154. } else if (signal !== undefined) {
  155. const onAbort = (): void => {
  156. this.detachInputSignal()
  157. this.cancel('workflow signal aborted')
  158. }
  159. this.inputSignal = signal
  160. this.inputSignalAbort = onAbort
  161. signal.addEventListener('abort', onAbort, { once: true })
  162. }
  163. }
  164. /**
  165. * Cancel the run: the worker is told (its hooks start throwing and the
  166. * script dies at its next await), the required signal shared by every child
  167. * start is aborted, and the grace timer
  168. * arms: a run still unsettled `disposeGraceMs` later force-settles
  169. * `cancelled` and its worker is TERMINATED. Idempotent; the first reason
  170. * wins.
  171. * @param reason - human-readable cause (default `'workflow cancelled'`).
  172. */
  173. cancel(reason?: string): void {
  174. // A settled run has nothing left to cancel, and a terminal source claimed
  175. // before its cleanup callbacks must exclude cancellation reentered by one
  176. // of those callbacks. Without the settled guard the
  177. // ordinary consumer path (await result, then dispose -> cancel) would arm
  178. // a grace timer nothing ever clears, pinning the run and its Worker
  179. // closure until the grace expires - a bounded leak per completed run.
  180. if (this.settled || this.terminalClaimed || this.cancelReason !== undefined) return
  181. this.cancelReason = reason ?? 'workflow cancelled'
  182. this.post(HostToWorkerType.Cancel, { reason: this.cancelReason })
  183. this.abortChildren(this.cancelReason)
  184. this.graceTimer = setTimeout(() => {
  185. // Cancellation already owns the race through cancelReason; close the
  186. // terminal boundary explicitly before observer teardown callbacks.
  187. this.terminalClaimed = true
  188. // The worker may no longer speak (it is about to be terminated): pair
  189. // every stranded start before the run settles, so ends precede
  190. // workflow/end.
  191. this.endStrandedAgents()
  192. this.settleResult(this.cancelledResult(this.hostStarted))
  193. void this.worker.terminate()
  194. }, this.disposeGraceMs)
  195. // unref'd: an armed grace timer must never hold the process open.
  196. this.graceTimer.unref()
  197. }
  198. /**
  199. * Cancel + bounded settle + termination. Host-drives every registered
  200. * child's disposal IMMEDIATELY — a wedged worker can relay no dispose RPC,
  201. * and deferring child teardown to the post-terminate reap would spend the
  202. * whole grace waiting for a quiescence that cannot start, then return with
  203. * the disposals still in flight — so child disposal overlaps the same
  204. * grace the worker gets to settle (the worker's own dispose RPCs join the
  205. * shared per-child disposal). Waits (at most the grace) for the result and
  206. * child quiescence, then terminates the worker unconditionally — the
  207. * thread never outlives its run — and reaps whatever children remain
  208. * (their disposal is contained, not awaited past the grace, the same
  209. * abandonment the seam documents for a slow-disposing child). Idempotent;
  210. * safe on every path.
  211. * @returns resolves when the run's resources are released or abandoned.
  212. */
  213. dispose(): Promise<void> {
  214. if (this.disposed !== undefined) return this.disposed
  215. // Claim the public transaction BEFORE its body invokes child/provider
  216. // disposal. A raw provider callback can reenter handle.dispose(); it must
  217. // join this promise rather than start a second traversal.
  218. const claimed = Promise.withResolvers<undefined>()
  219. this.disposed = claimed.promise
  220. void (async () => {
  221. this.detachInputSignal()
  222. this.cancel('workflow disposed')
  223. // cancel() deliberately becomes a no-op after terminal settlement, but
  224. // disposal still owns every registered child. Reap independently so an
  225. // already-settled workflow cannot wait on child quiescence before it has
  226. // started the surviving children's disposals. On an unsettled run this
  227. // joins the cancel path through the per-call cancellation/disposal gates.
  228. this.reapChildren('workflow disposed')
  229. await Promise.race([
  230. (async () => {
  231. await this.result
  232. await this.childQuiescence()
  233. })(),
  234. sleep(this.disposeGraceMs),
  235. ])
  236. await this.worker.terminate()
  237. this.reapChildren('workflow disposed')
  238. })().then(
  239. () => { claimed.resolve(undefined) },
  240. /* v8 ignore next -- result/quiescence never reject and Worker.terminate is the only external promise */
  241. (error: unknown) => { claimed.reject(error) },
  242. )
  243. return this.disposed
  244. }
  245. /** Post one message to the worker (payload looked up from the tag's map entry), tolerating a thread that is already gone. */
  246. private post<T extends HostToWorkerType>(type: T, payload: HostToWorkerPayloads[T]): void {
  247. if (this.workerGone || this.workerDeathObserved) return
  248. try {
  249. this.worker.postMessage({ type, ...payload })
  250. } catch (error: unknown) {
  251. // Only a teardown race can land here (every engine message is JSON
  252. // data, so serialization cannot fail); there is nothing left to
  253. // deliver to — log and move on.
  254. /* v8 ignore next -- postMessage teardown race (a throw between exit and its event): not constructible in-process */
  255. this.ctx.logger.warn(`workflow-worker-thread: postMessage failed: ${renderThrown(error)}`)
  256. }
  257. }
  258. private onMessage(message: WorkerToHostMessage): void {
  259. // Node may emit `error`, then deliver an already-queued `message`, then
  260. // emit `exit`. The first death signal is the host's logical delivery
  261. // barrier: nothing arriving afterward may create a child, narrate after
  262. // workflow/end, or compete with the chosen outcome.
  263. if (this.workerDeathObserved) return
  264. switch (message.type) {
  265. case WorkerToHostType.Ready:
  266. this.post(HostToWorkerType.Go, {})
  267. break
  268. case WorkerToHostType.Phase:
  269. // Post-cancel narration is suppressed host-side: worker-side the
  270. // hooks throw once the cancel message is PROCESSED, but narration
  271. // already in flight (or emitted while the cancel crossed the
  272. // boundary) must not reach observers — nothing is emitted after
  273. // cancel() returns.
  274. if (this.cancelReason === undefined) this.observer.phase(message.title)
  275. break
  276. case WorkerToHostType.Log:
  277. if (this.cancelReason === undefined) this.observer.log(message.message)
  278. break
  279. case WorkerToHostType.AgentStart:
  280. this.liveAgents.set(message.info.seq, message.info)
  281. this.observer.agentStart(message.info)
  282. break
  283. case WorkerToHostType.AgentEnd:
  284. // NOT suppressed on cancel: cancelled children report their paired
  285. // agent-end with outcome 'cancelled'. The gate (with the termination
  286. // paths' synthesis) is what makes the one-pair-per-started-child
  287. // contract hold on every stop path.
  288. this.endAgent(message.info)
  289. break
  290. case WorkerToHostType.ChildStart:
  291. this.onChildStart(message.callId, message.request)
  292. break
  293. case WorkerToHostType.ChildDispose:
  294. this.onChildDispose(message.callId)
  295. break
  296. case WorkerToHostType.Result:
  297. this.onResult(message.result)
  298. break
  299. /* v8 ignore next 2 -- closed engine-owned union; the arm only makes adding a message type a compile error */
  300. default:
  301. assertNever(message, 'worker-to-host message')
  302. }
  303. }
  304. /** Why a ready provider result may no longer be admitted to the worker. */
  305. private childAdmissionFailure(): { reason: string; rendered: string } | undefined {
  306. if (this.cancelReason !== undefined) {
  307. return { reason: this.cancelReason, rendered: `workflow run cancelled: ${this.cancelReason}` }
  308. }
  309. if (this.workerDeathObserved) {
  310. return { reason: 'workflow worker gone', rendered: 'workflow worker is no longer available' }
  311. }
  312. if (this.terminalClaimed) {
  313. return { reason: 'workflow settled', rendered: 'workflow run already settled' }
  314. }
  315. return undefined
  316. }
  317. private onChildStart(callId: number, request: ChildStartRequest): void {
  318. const initialFailure = this.childAdmissionFailure()
  319. if (initialFailure !== undefined) {
  320. // Refuse after a terminal boundary: a child must never start on an
  321. // already-aborted signal (a provider subscribing only to future abort
  322. // events would never observe it).
  323. this.post(HostToWorkerType.ChildStartError, { callId, rendered: initialFailure.rendered })
  324. return
  325. }
  326. this.hostStarted += 1
  327. const task = this.startChild(callId, request)
  328. this.pendingStarts.add(task)
  329. void task.then(
  330. () => { this.finishPendingStart(task) },
  331. /* v8 ignore next -- startChild contains provider and cleanup failures */
  332. () => { this.finishPendingStart(task) },
  333. )
  334. }
  335. /** Await one provider-owned startup transaction and publish only while admitted. */
  336. private async startChild(callId: number, request: ChildStartRequest): Promise<void> {
  337. let run: SubagentRun
  338. try {
  339. run = await this.subagents.start(this.provider, {
  340. prompt: [{ type: 'text', text: request.prompt }],
  341. parent: this.parent,
  342. signal: this.controller.signal,
  343. ...request.schema !== undefined ? { outputSchema: request.schema } : {},
  344. ...request.provider !== undefined || request.model !== undefined
  345. ? {
  346. agentOptions: {
  347. ...request.provider !== undefined ? { provider: request.provider } : {},
  348. ...request.model !== undefined ? { model: request.model } : {},
  349. },
  350. }
  351. : {},
  352. })
  353. } catch (error: unknown) {
  354. const failure = this.childAdmissionFailure()
  355. this.post(HostToWorkerType.ChildStartError, {
  356. callId,
  357. rendered: failure?.rendered ?? renderThrown(error),
  358. })
  359. return
  360. }
  361. const failure = this.childAdmissionFailure()
  362. if (failure !== undefined) {
  363. this.post(HostToWorkerType.ChildStartError, { callId, rendered: failure.rendered })
  364. try {
  365. await run.dispose()
  366. } catch (error: unknown) {
  367. this.ctx.logger.warn(`workflow-worker-thread: refused child dispose failed: ${renderThrown(error)}`)
  368. }
  369. return
  370. }
  371. const record: ChildRecord = { run }
  372. this.children.set(callId, record)
  373. // Attach result forwarding before publishing the child handle. Because the
  374. // callback itself runs in a later microtask, ChildStarted is still posted
  375. // first even for an already-settled scripted provider.
  376. const forwardResult = run.result.then<() => void, () => void>(
  377. (result) => {
  378. try {
  379. const snapshot = snapshotJsonValue<ChildResult>({
  380. output: result.output,
  381. ...result.structured !== undefined ? { structured: result.structured } : {},
  382. stopReason: result.stopReason,
  383. })
  384. if (snapshot === undefined) throw new TypeError('child result is not losslessly JSON-serializable')
  385. return () => { this.post(HostToWorkerType.ChildSettled, { callId, result: snapshot }) }
  386. } catch (error: unknown) {
  387. const rendered = `workflow child result could not cross the worker boundary: ${renderThrown(error)}`
  388. return () => { this.post(HostToWorkerType.ChildFailed, { callId, rendered }) }
  389. }
  390. },
  391. (error: unknown) => {
  392. const rendered = renderThrown(error)
  393. return () => { this.post(HostToWorkerType.ChildFailed, { callId, rendered }) }
  394. },
  395. )
  396. this.post(HostToWorkerType.ChildStarted, { callId, childId: run.id })
  397. void forwardResult.then((forward) => { forward() })
  398. }
  399. private onChildDispose(callId: number): void {
  400. const record = this.children.get(callId)
  401. if (record === undefined) {
  402. // Already disposed host-side (a dispose() drive or a death reap beat
  403. // the RPC) — the ack is still owed (the worker-side wrapper awaits it).
  404. this.post(HostToWorkerType.ChildDisposed, { callId })
  405. return
  406. }
  407. // disposeChild never rejects (containment is inside), so the ack always follows.
  408. void this.disposeChild(callId, record).then(() => { this.post(HostToWorkerType.ChildDisposed, { callId }) })
  409. }
  410. /**
  411. * Start (or join) one registered child's disposal; the registry entry
  412. * leaves when it settles. Memoized per callId: the worker's dispose RPC,
  413. * the dispose() host drive, and the reap can all land on the same child —
  414. * the child's `dispose()` runs once and every caller awaits that one
  415. * settlement. A rejection is contained (the subagent seam's dispose() is
  416. * not supposed to reject, but a backend that does anyway must not break
  417. * quiescence): logged, and the child still leaves the registry.
  418. * @param callId - the child's registry key.
  419. * @param record - the registered child (the caller looked it up).
  420. * @returns resolves when the disposal settled either way; never rejects.
  421. */
  422. private disposeChild(callId: number, record: ChildRecord): Promise<void> {
  423. if (record.disposal !== undefined) return record.disposal
  424. record.disposal = Promise.resolve()
  425. .then(() => record.run.dispose())
  426. .catch((error: unknown) => {
  427. this.ctx.logger.warn(`workflow-worker-thread: child dispose failed: ${renderThrown(error)}`)
  428. })
  429. .then(() => { this.finishChild(callId) })
  430. return record.disposal
  431. }
  432. /** Drop a child record and release quiescence waiters when all work ends. */
  433. private finishChild(callId: number): void {
  434. this.children.delete(callId)
  435. this.notifyChildQuiescence()
  436. }
  437. /** Retire one provider startup transaction. */
  438. private finishPendingStart(task: Promise<void>): void {
  439. this.pendingStarts.delete(task)
  440. this.notifyChildQuiescence()
  441. }
  442. /** Release waiters only after both pending starts and published children end. */
  443. private notifyChildQuiescence(): void {
  444. if (this.children.size !== 0 || this.pendingStarts.size !== 0) return
  445. for (const waiter of this.quiescenceWaiters.splice(0)) waiter()
  446. }
  447. /** Resolves once every pending start and published child has reached quiescence. */
  448. private childQuiescence(): Promise<void> {
  449. if (this.children.size === 0 && this.pendingStarts.size === 0) return Promise.resolve()
  450. return new Promise((resolve) => { this.quiescenceWaiters.push(resolve) })
  451. }
  452. /** Abort + dispose every registered child (worker death / final teardown); disposal is contained, not awaited. */
  453. private reapChildren(reason: string): void {
  454. this.abortChildren(this.cancelReason ?? reason)
  455. for (const [callId, record] of [...this.children]) {
  456. void this.disposeChild(callId, record)
  457. }
  458. }
  459. /** Abort the one canonical signal shared by pending and published children. */
  460. private abortChildren(reason: string): void {
  461. if (!this.controller.signal.aborted) this.controller.abort(reason)
  462. }
  463. private onResult(result: WorkflowResult): void {
  464. // The owned worker session sends one Result. Keep a late duplicate or a
  465. // Result queued behind another terminal source completely side-effect-free.
  466. if (this.terminalClaimed) return
  467. // First-wins is decided when the Result message reaches the host. If no
  468. // external cancellation was already in flight, this result won. Reaping a
  469. // stray child below may synchronously reenter cancel() through provider
  470. // callbacks, but that internal post-result cleanup must not retroactively
  471. // rewrite the worker result that arrived first.
  472. const cancellationWasRequested = this.cancelReason !== undefined
  473. // Claim before settlement cleanup invokes provider disposal. Once Result
  474. // won, a later cancellation cannot rewrite it.
  475. this.terminalClaimed = true
  476. // Abort pending starts and begin disposing published children before the
  477. // workflow becomes externally settled. Cleanup remains independently
  478. // tracked by childQuiescence and the holder's dispose().
  479. this.reapChildren('workflow settled')
  480. if (!cancellationWasRequested) {
  481. this.settleResult(result)
  482. return
  483. }
  484. if (result.stopReason !== 'cancelled') {
  485. // The script settled while our cancel was crossing the thread boundary
  486. // — the seam-visible result had NOT settled when cancellation was
  487. // requested, so report cancelled (the vm drive()'s post-settle check,
  488. // relocated to the receiving side of the race).
  489. this.settleResult(this.cancelledResult(result.agentsStarted))
  490. return
  491. }
  492. this.settleResult(result)
  493. }
  494. /** Process an error/messageerror/exit signal; `exit` also performs the final disposal sweep. */
  495. private onWorkerDeath(message: string, isExit: boolean): void {
  496. if (!this.workerDeathObserved) {
  497. // Close message admission BEFORE cleanup callbacks: Node can deliver a
  498. // message queued before the crash after its `error` event. Treating the
  499. // first death signal as a logical barrier prevents that late message
  500. // from creating work or narrating after workflow/end.
  501. this.workerDeathObserved = true
  502. const outcomeWasClaimed = this.terminalClaimed
  503. const cancellationWasRequested = this.cancelReason !== undefined
  504. // When death is itself the terminal source, claim BEFORE child reap or
  505. // synthesized observer callbacks. Either can reenter cancel(); a death
  506. // that arrived first remains an error, while a cancellation already
  507. // accepted before death remains cancelled. If Result/grace already won,
  508. // preserve it while still performing prompt failure-time cleanup.
  509. if (!outcomeWasClaimed) this.terminalClaimed = true
  510. if (this.children.size > 0 || this.pendingStarts.size > 0) this.reapChildren('workflow worker gone')
  511. this.endStrandedAgents()
  512. if (!outcomeWasClaimed) {
  513. if (cancellationWasRequested) {
  514. this.settleResult(this.cancelledResult(this.hostStarted))
  515. } else {
  516. this.settleResult({ value: null, stopReason: 'error', error: message, agentsStarted: this.hostStarted })
  517. }
  518. }
  519. }
  520. if (!isExit) return
  521. // `error` is not Node's physical delivery barrier: a queued message may
  522. // precede `exit`. Admission is already closed, so this final sweep only
  523. // joins/starts disposal for registry survivors; it deliberately does not
  524. // repeat explicit provider cancellation.
  525. for (const [callId, record] of [...this.children]) void this.disposeChild(callId, record)
  526. this.endStrandedAgents()
  527. }
  528. /**
  529. * The single agent-end emission gate: forwards `end` iff its start is still
  530. * unpaired in the ledger, so every forwarded `workflow/agent-start` gets
  531. * EXACTLY one `workflow/agent-end` — the worker's own report where it can
  532. * speak, a host-synthesized one where it cannot ({@link endStrandedAgents}).
  533. * @param end - the settlement to emit (worker-reported or synthesized).
  534. */
  535. private endAgent(end: WorkflowAgentEndInfo): void {
  536. /* v8 ignore next -- a real end still in flight across the grace force-settle: not orderable in-process */
  537. if (!this.liveAgents.delete(end.seq)) return
  538. this.observer.agentEnd(end)
  539. }
  540. /**
  541. * Synthesize the missing `agent-end` for every started-but-unpaired agent,
  542. * outcome `'cancelled'`: the reap cancels every child, and a real
  543. * settlement racing the force-settle loses to that already-started external
  544. * cancellation. The atomic terminal boundaries in {@link onResult} and
  545. * {@link onWorkerDeath} deliberately exclude teardown callbacks as contenders.
  546. * Called where the worker can no longer speak (the grace force-settle,
  547. * worker death, physical exit). When grace/death is the terminal source it
  548. * runs before settleResult, so already-known pairs precede `workflow/end`;
  549. * after an earlier Result, exit cleanup may close a survivor afterward.
  550. * The ledger preserves exactly-once pairing in both orders.
  551. */
  552. private endStrandedAgents(): void {
  553. for (const info of [...this.liveAgents.values()]) {
  554. this.endAgent({ ...info, outcome: 'cancelled' })
  555. }
  556. }
  557. private cancelledResult(agentsStarted: number): WorkflowResult {
  558. // cancel() is the only writer of cancelReason and every caller checks it
  559. // first; the fallback guards the type, not a reachable path.
  560. /* v8 ignore next */
  561. const reason = this.cancelReason ?? 'workflow cancelled'
  562. return { value: null, stopReason: 'cancelled', error: `workflow run cancelled: ${reason}`, agentsStarted }
  563. }
  564. /** Remove the exact abort callback installed on the caller's start signal. */
  565. private detachInputSignal(): void {
  566. const signal = this.inputSignal
  567. const onAbort = this.inputSignalAbort
  568. if (signal === undefined || onAbort === undefined) return
  569. this.inputSignal = undefined
  570. this.inputSignalAbort = undefined
  571. signal.removeEventListener('abort', onAbort)
  572. }
  573. /** First settle wins; disarms the grace timer and releases the caller signal. */
  574. private settleResult(result: WorkflowResult): void {
  575. // Every current terminal source claims ownership before calling here; keep
  576. // the fallback local so a future caller cannot resolve twice.
  577. /* v8 ignore next -- defensive fallback outside the claimed state machine */
  578. if (this.settled) return
  579. this.terminalClaimed = true
  580. this.settled = true
  581. this.detachInputSignal()
  582. clearTimeout(this.graceTimer)
  583. this.settleResolve(result)
  584. }
  585. }
  586. /** A plain timer sleep (the dispose grace); unref'd so it never holds the process open. */
  587. function sleep(ms: number): Promise<void> {
  588. return new Promise((resolve) => {
  589. const timer = setTimeout(resolve, ms)
  590. timer.unref()
  591. })
  592. }