host.ts 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625
  1. /**
  2. * Host side of one workflow run. The first worker result, unexpected death, or
  3. * cancellation-grace expiry owns settlement and closes message admission.
  4. * Pending starts share one abort signal; published children share idempotent
  5. * cleanup, and quiescence waits for both while synthesizing any missing end events.
  6. * @module @deepseek-ai/dsh-workflow-worker-thread/host
  7. */
  8. import { tmpdir } from 'node:os'
  9. import { Worker } from 'node:worker_threads'
  10. import type { WorkerOptions } from 'node:worker_threads'
  11. import { fileURLToPath } from 'node:url'
  12. import type { Context } from '@deepseek-ai/cordis'
  13. import type { Agent } from '@deepseek-ai/dsh-agent'
  14. import { assertNever, snapshotJsonValue } from '@deepseek-ai/dsh-util-values'
  15. import type SubagentRuntime from '@deepseek-ai/dsh-subagent'
  16. import type { SubagentRun } from '@deepseek-ai/dsh-subagent'
  17. import type { WorkflowAgentEndInfo, WorkflowAgentInfo, WorkflowMeta, WorkflowResult, WorkflowRun, WorkflowRunId } from '@deepseek-ai/dsh-workflow'
  18. import { renderThrown } from './realm.ts'
  19. import type { ExecutionObserver } from './runtime.ts'
  20. import { HostToWorkerType, WorkerToHostType } from './protocol.ts'
  21. import type { HostToWorkerPayloads, WorkerToHostMessage } from './protocol.ts'
  22. import type { ChildResult, ChildStartRequest, WorkerInit } from './types.ts'
  23. /** One published child and its shared quiescent-disposal transaction. */
  24. interface ChildRecord {
  25. readonly run: SubagentRun
  26. disposal?: Promise<void>
  27. }
  28. /**
  29. * The scrubbed worker environment: no ambient credentials, no loader flags, and deliberately no
  30. * proxy policy. A worker thread does not inherit the host's global dispatcher, so a workflow's own
  31. * requests go direct — the alternative is handing the worker a proxy URL that may carry
  32. * `user:password`, and this worker executes the model-authored script body. That is the same
  33. * containment the code runtime keeps, and `docs/defensive-patterns.md` requires it.
  34. * Windows derives `os.tmpdir()` from `TMP`/`TEMP` and falls back to the
  35. * literal relative path `undefined\temp` when the environment is empty, so
  36. * tsx's transform cache would land in a cwd-relative `undefined/temp`
  37. * directory; the host's real temp path (not a credential) is injected there.
  38. * The unbuilt shape additionally forwards `TSX_TSCONFIG_PATH` for path
  39. * resolution.
  40. * @param platform - host platform; overridable so tests exercise both peer arms.
  41. * @param tsconfigPath - the tsconfig pin to forward; only the unbuilt caller
  42. * passes one, so the built worker never observes the host's pin.
  43. * @returns the scrubbed worker environment object.
  44. */
  45. export function workerSpawnEnv(
  46. platform: NodeJS.Platform = process.platform,
  47. tsconfigPath?: string,
  48. ): NodeJS.ProcessEnv {
  49. const env: NodeJS.ProcessEnv = {}
  50. if (platform === 'win32') {
  51. const tmp = tmpdir()
  52. env.TMP = tmp
  53. env.TEMP = tmp
  54. }
  55. if (tsconfigPath !== undefined) env.TSX_TSCONFIG_PATH = tsconfigPath
  56. return env
  57. }
  58. /**
  59. * Resolve a built worker bundle or an unbuilt bootstrap that installs both tsx
  60. * transforms inside the worker. Both shapes clear `execArgv` and the ambient
  61. * environment (the worker only sees the platform temp path and, unbuilt,
  62. * `TSX_TSCONFIG_PATH`).
  63. * @param init - the run payload, passed as `workerData`.
  64. * @returns the entry path or URL and the Worker options to spawn it with.
  65. */
  66. function resolveWorkerSpawn(init: WorkerInit): { entry: string | URL; options: WorkerOptions } {
  67. /* v8 ignore next 3 -- the built-output arm: tests always run unbuilt (src/); the built-worker e2e exercises this shape for real */
  68. if (!import.meta.url.endsWith('.ts')) {
  69. return { entry: fileURLToPath(new URL('./worker.cjs', import.meta.url)), options: { workerData: init, env: workerSpawnEnv(), execArgv: [] } }
  70. }
  71. // Resolve tsx only for unbuilt consumers and install it before importing TS.
  72. const workerEntry = new URL('./worker.ts', import.meta.url)
  73. const tsxEsmApiEntry = import.meta.resolve('tsx/esm/api')
  74. const tsxCjsApiEntry = import.meta.resolve('tsx/cjs/api')
  75. const bootstrap = [
  76. `import { register as registerEsm } from ${JSON.stringify(tsxEsmApiEntry)}`,
  77. `import { register as registerCjs } from ${JSON.stringify(tsxCjsApiEntry)}`,
  78. 'registerCjs()',
  79. 'registerEsm()',
  80. `await import(${JSON.stringify(workerEntry.href)})`,
  81. ].join('\n')
  82. return {
  83. entry: new URL(`data:text/javascript,${encodeURIComponent(bootstrap)}`),
  84. options: {
  85. workerData: init,
  86. env: workerSpawnEnv(undefined, process.env.TSX_TSCONFIG_PATH),
  87. execArgv: [],
  88. },
  89. }
  90. }
  91. /**
  92. * One live worker-engine run — the seam's {@link WorkflowRun}, returned by
  93. * `start()` directly. Owns the Worker, the child registry, and the result
  94. * settlement; `result` never rejects. `meta` is trusted same-process data
  95. * borrowed as immutable by the handle and lifecycle events. The holder-bound
  96. * SubagentRuntime handle is captured before the
  97. * engine returns this run, so unloading the engine removes only the ability to
  98. * start another workflow; this run can still start and clean up its children.
  99. */
  100. export class WorkerRun implements WorkflowRun {
  101. /** Settles exactly once with the run's outcome; never rejects. */
  102. readonly result: Promise<WorkflowResult>
  103. private settleResolve!: (result: WorkflowResult) => void
  104. private settled = false
  105. /** A Result/death/grace outcome atomically won before teardown callbacks. */
  106. private terminalClaimed = false
  107. /** The first death signal closes worker-message admission and owns failure-time cleanup. */
  108. private workerDeathObserved = false
  109. private cancelReason: string | undefined
  110. private graceTimer: NodeJS.Timeout | undefined
  111. private readonly worker: Worker
  112. /** Set on `exit`: the thread is gone, so posting has nowhere to go. */
  113. private workerGone = false
  114. /** Accepted `child-start` messages — the terminate-path `agentsStarted` (see module doc). */
  115. private hostStarted = 0
  116. /** Published children by callId; an entry leaves only after disposal settles. */
  117. private readonly children = new Map<number, ChildRecord>()
  118. /** Provider starts that have not yet fulfilled or rejected. */
  119. private readonly pendingStarts = new Set<Promise<void>>()
  120. /** Started-but-not-ended agents by seq — the pairing ledger the HOST guarantees (see {@link endAgent}). */
  121. private readonly liveAgents = new Map<number, WorkflowAgentInfo>()
  122. private readonly quiescenceWaiters: (() => void)[] = []
  123. /** The per-run abort fanout every child start request carries. */
  124. private readonly controller = new AbortController()
  125. /** External start signal and the exact callback installed on it, retained only until first settle/teardown. */
  126. private inputSignal: AbortSignal | undefined
  127. private inputSignalAbort: (() => void) | undefined
  128. private disposed: Promise<void> | undefined
  129. constructor(
  130. private readonly ctx: Context,
  131. private readonly subagents: SubagentRuntime,
  132. readonly id: WorkflowRunId,
  133. readonly meta: WorkflowMeta,
  134. private readonly parent: Agent,
  135. init: WorkerInit,
  136. private readonly provider: string,
  137. private readonly disposeGraceMs: number,
  138. private readonly observer: ExecutionObserver,
  139. signal: AbortSignal | undefined,
  140. ) {
  141. this.result = new Promise<WorkflowResult>((resolve) => { this.settleResolve = resolve })
  142. // workerData rides the structured clone: args are plain JSON by the seam
  143. // contract, so the clone is total and doubles as the caller-isolation
  144. // copy (a clone failure throws loud out of start()).
  145. const { entry, options } = resolveWorkerSpawn(init)
  146. this.worker = new Worker(entry, options)
  147. this.worker.on('message', (message: WorkerToHostMessage) => { this.onMessage(message) })
  148. this.worker.on('error', (error) => { this.onWorkerDeath(`workflow worker failed: ${renderThrown(error)}`, false) })
  149. /* v8 ignore next -- messageerror: not constructible from the engine's own protocol (every payload is JSON data) */
  150. this.worker.on('messageerror', (error) => { this.onWorkerDeath(`workflow worker message failed to deserialize: ${renderThrown(error)}`, false) })
  151. this.worker.on('exit', (code) => {
  152. this.workerGone = true
  153. this.onWorkerDeath(`workflow worker exited before the run settled (exit code ${code})`, true)
  154. })
  155. if (signal?.aborted) {
  156. this.cancel('workflow start signal already aborted')
  157. } else if (signal !== undefined) {
  158. const onAbort = (): void => {
  159. this.detachInputSignal()
  160. this.cancel('workflow signal aborted')
  161. }
  162. this.inputSignal = signal
  163. this.inputSignalAbort = onAbort
  164. signal.addEventListener('abort', onAbort, { once: true })
  165. }
  166. }
  167. /**
  168. * Cancel the run: the worker is told (its hooks start throwing and the
  169. * script dies at its next await), the required signal shared by every child
  170. * start is aborted, and the grace timer
  171. * arms: a run still unsettled `disposeGraceMs` later force-settles
  172. * `cancelled` and its worker is TERMINATED. Idempotent; the first reason
  173. * wins.
  174. * @param reason - human-readable cause (default `'workflow cancelled'`).
  175. */
  176. cancel(reason?: string): void {
  177. // A settled run has nothing left to cancel, and a terminal source claimed
  178. // before its cleanup callbacks must exclude cancellation reentered by one
  179. // of those callbacks. Without the settled guard the
  180. // ordinary consumer path (await result, then dispose -> cancel) would arm
  181. // a grace timer nothing ever clears, pinning the run and its Worker
  182. // closure until the grace expires - a bounded leak per completed run.
  183. if (this.settled || this.terminalClaimed || this.cancelReason !== undefined) return
  184. this.cancelReason = reason ?? 'workflow cancelled'
  185. this.post(HostToWorkerType.Cancel, { reason: this.cancelReason })
  186. this.abortChildren(this.cancelReason)
  187. this.graceTimer = setTimeout(() => {
  188. // Cancellation already owns the race through cancelReason; close the
  189. // terminal boundary explicitly before observer teardown callbacks.
  190. this.terminalClaimed = true
  191. // The worker may no longer speak (it is about to be terminated): pair
  192. // every stranded start before the run settles, so ends precede
  193. // workflow/end.
  194. this.endStrandedAgents()
  195. this.settleResult(this.cancelledResult(this.hostStarted))
  196. void this.worker.terminate()
  197. }, this.disposeGraceMs)
  198. // unref'd: an armed grace timer must never hold the process open.
  199. this.graceTimer.unref()
  200. }
  201. /**
  202. * Cancel + bounded settle + termination. Host-drives every registered
  203. * child's disposal IMMEDIATELY — a wedged worker can relay no dispose RPC,
  204. * and deferring child teardown to the post-terminate reap would spend the
  205. * whole grace waiting for a quiescence that cannot start, then return with
  206. * the disposals still in flight — so child disposal overlaps the same
  207. * grace the worker gets to settle (the worker's own dispose RPCs join the
  208. * shared per-child disposal). Waits (at most the grace) for the result and
  209. * child quiescence, then terminates the worker unconditionally — the
  210. * thread never outlives its run — and reaps whatever children remain
  211. * (their disposal is contained, not awaited past the grace, the same
  212. * abandonment the seam documents for a slow-disposing child). Idempotent;
  213. * safe on every path.
  214. * @returns resolves when the run's resources are released or abandoned.
  215. */
  216. dispose(): Promise<void> {
  217. if (this.disposed !== undefined) return this.disposed
  218. // Claim the public transaction BEFORE its body invokes child/provider
  219. // disposal. A raw provider callback can reenter handle.dispose(); it must
  220. // join this promise rather than start a second traversal.
  221. const claimed = Promise.withResolvers<undefined>()
  222. this.disposed = claimed.promise
  223. void (async () => {
  224. this.detachInputSignal()
  225. this.cancel('workflow disposed')
  226. // cancel() deliberately becomes a no-op after terminal settlement, but
  227. // disposal still owns every registered child. Reap independently so an
  228. // already-settled workflow cannot wait on child quiescence before it has
  229. // started the surviving children's disposals. On an unsettled run this
  230. // joins the cancel path through the per-call cancellation/disposal gates.
  231. this.reapChildren('workflow disposed')
  232. await Promise.race([
  233. (async () => {
  234. await this.result
  235. await this.childQuiescence()
  236. })(),
  237. sleep(this.disposeGraceMs),
  238. ])
  239. await this.worker.terminate()
  240. this.reapChildren('workflow disposed')
  241. })().then(
  242. () => { claimed.resolve(undefined) },
  243. /* v8 ignore next -- result/quiescence never reject and Worker.terminate is the only external promise */
  244. (error: unknown) => { claimed.reject(error) },
  245. )
  246. return this.disposed
  247. }
  248. /** Post one message to the worker (payload looked up from the tag's map entry), tolerating a thread that is already gone. */
  249. private post<T extends HostToWorkerType>(type: T, payload: HostToWorkerPayloads[T]): void {
  250. if (this.workerGone || this.workerDeathObserved) return
  251. try {
  252. this.worker.postMessage({ type, ...payload })
  253. } catch (error: unknown) {
  254. // Only a teardown race can land here (every engine message is JSON
  255. // data, so serialization cannot fail); there is nothing left to
  256. // deliver to — log and move on.
  257. /* v8 ignore next -- postMessage teardown race (a throw between exit and its event): not constructible in-process */
  258. this.ctx.logger.warn(`workflow-worker-thread: postMessage failed: ${renderThrown(error)}`)
  259. }
  260. }
  261. private onMessage(message: WorkerToHostMessage): void {
  262. // Node may emit `error`, then deliver an already-queued `message`, then
  263. // emit `exit`. The first death signal is the host's logical delivery
  264. // barrier: nothing arriving afterward may create a child, narrate after
  265. // workflow/end, or compete with the chosen outcome.
  266. if (this.workerDeathObserved) return
  267. switch (message.type) {
  268. case WorkerToHostType.Ready:
  269. this.post(HostToWorkerType.Go, {})
  270. break
  271. case WorkerToHostType.Phase:
  272. // Post-cancel narration is suppressed host-side: worker-side the
  273. // hooks throw once the cancel message is PROCESSED, but narration
  274. // already in flight (or emitted while the cancel crossed the
  275. // boundary) must not reach observers — nothing is emitted after
  276. // cancel() returns.
  277. if (this.cancelReason === undefined) this.observer.phase(message.title)
  278. break
  279. case WorkerToHostType.Log:
  280. if (this.cancelReason === undefined) this.observer.log(message.message)
  281. break
  282. case WorkerToHostType.AgentStart:
  283. this.liveAgents.set(message.info.seq, message.info)
  284. this.observer.agentStart(message.info)
  285. break
  286. case WorkerToHostType.AgentEnd:
  287. // NOT suppressed on cancel: cancelled children report their paired
  288. // agent-end with outcome 'cancelled'. The gate (with the termination
  289. // paths' synthesis) is what makes the one-pair-per-started-child
  290. // contract hold on every stop path.
  291. this.endAgent(message.info)
  292. break
  293. case WorkerToHostType.ChildStart:
  294. this.onChildStart(message.callId, message.request)
  295. break
  296. case WorkerToHostType.ChildDispose:
  297. this.onChildDispose(message.callId)
  298. break
  299. case WorkerToHostType.Result:
  300. this.onResult(message.result)
  301. break
  302. /* v8 ignore next 2 -- closed engine-owned union; the arm only makes adding a message type a compile error */
  303. default:
  304. assertNever(message, 'worker-to-host message')
  305. }
  306. }
  307. /** Why a ready provider result may no longer be admitted to the worker. */
  308. private childAdmissionFailure(): { reason: string; rendered: string } | undefined {
  309. if (this.cancelReason !== undefined) {
  310. return { reason: this.cancelReason, rendered: `workflow run cancelled: ${this.cancelReason}` }
  311. }
  312. if (this.workerDeathObserved) {
  313. return { reason: 'workflow worker gone', rendered: 'workflow worker is no longer available' }
  314. }
  315. if (this.terminalClaimed) {
  316. return { reason: 'workflow settled', rendered: 'workflow run already settled' }
  317. }
  318. return undefined
  319. }
  320. private onChildStart(callId: number, request: ChildStartRequest): void {
  321. const initialFailure = this.childAdmissionFailure()
  322. if (initialFailure !== undefined) {
  323. // Refuse after a terminal boundary: a child must never start on an
  324. // already-aborted signal (a provider subscribing only to future abort
  325. // events would never observe it).
  326. this.post(HostToWorkerType.ChildStartError, { callId, rendered: initialFailure.rendered })
  327. return
  328. }
  329. this.hostStarted += 1
  330. const task = this.startChild(callId, request)
  331. this.pendingStarts.add(task)
  332. void task.then(
  333. () => { this.finishPendingStart(task) },
  334. /* v8 ignore next -- startChild contains provider and cleanup failures */
  335. () => { this.finishPendingStart(task) },
  336. )
  337. }
  338. /** Await one provider-owned startup transaction and publish only while admitted. */
  339. private async startChild(callId: number, request: ChildStartRequest): Promise<void> {
  340. let run: SubagentRun
  341. try {
  342. run = await this.subagents.start(this.provider, {
  343. prompt: [{ type: 'text', text: request.prompt }],
  344. parent: this.parent,
  345. signal: this.controller.signal,
  346. ...request.schema !== undefined ? { outputSchema: request.schema } : {},
  347. ...request.provider !== undefined || request.model !== undefined
  348. ? {
  349. agentOptions: {
  350. ...request.provider !== undefined ? { provider: request.provider } : {},
  351. ...request.model !== undefined ? { model: request.model } : {},
  352. },
  353. }
  354. : {},
  355. })
  356. } catch (error: unknown) {
  357. const failure = this.childAdmissionFailure()
  358. this.post(HostToWorkerType.ChildStartError, {
  359. callId,
  360. rendered: failure?.rendered ?? renderThrown(error),
  361. })
  362. return
  363. }
  364. const failure = this.childAdmissionFailure()
  365. if (failure !== undefined) {
  366. this.post(HostToWorkerType.ChildStartError, { callId, rendered: failure.rendered })
  367. try {
  368. await run.dispose()
  369. } catch (error: unknown) {
  370. this.ctx.logger.warn(`workflow-worker-thread: refused child dispose failed: ${renderThrown(error)}`)
  371. }
  372. return
  373. }
  374. const record: ChildRecord = { run }
  375. this.children.set(callId, record)
  376. // Attach result forwarding before publishing the child handle. Because the
  377. // callback itself runs in a later microtask, ChildStarted is still posted
  378. // first even for an already-settled scripted provider.
  379. const forwardResult = run.result.then<() => void, () => void>(
  380. (result) => {
  381. try {
  382. const snapshot = snapshotJsonValue<ChildResult>({
  383. output: result.output,
  384. ...result.structured !== undefined ? { structured: result.structured } : {},
  385. stopReason: result.stopReason,
  386. })
  387. if (snapshot === undefined) throw new TypeError('child result is not losslessly JSON-serializable')
  388. return () => { this.post(HostToWorkerType.ChildSettled, { callId, result: snapshot }) }
  389. } catch (error: unknown) {
  390. const rendered = `workflow child result could not cross the worker boundary: ${renderThrown(error)}`
  391. return () => { this.post(HostToWorkerType.ChildFailed, { callId, rendered }) }
  392. }
  393. },
  394. (error: unknown) => {
  395. const rendered = renderThrown(error)
  396. return () => { this.post(HostToWorkerType.ChildFailed, { callId, rendered }) }
  397. },
  398. )
  399. this.post(HostToWorkerType.ChildStarted, { callId, childId: run.id })
  400. void forwardResult.then((forward) => { forward() })
  401. }
  402. private onChildDispose(callId: number): void {
  403. const record = this.children.get(callId)
  404. if (record === undefined) {
  405. // Already disposed host-side (a dispose() drive or a death reap beat
  406. // the RPC) — the ack is still owed (the worker-side wrapper awaits it).
  407. this.post(HostToWorkerType.ChildDisposed, { callId })
  408. return
  409. }
  410. // disposeChild never rejects (containment is inside), so the ack always follows.
  411. void this.disposeChild(callId, record).then(() => { this.post(HostToWorkerType.ChildDisposed, { callId }) })
  412. }
  413. /**
  414. * Start (or join) one registered child's disposal; the registry entry
  415. * leaves when it settles. Memoized per callId: the worker's dispose RPC,
  416. * the dispose() host drive, and the reap can all land on the same child —
  417. * the child's `dispose()` runs once and every caller awaits that one
  418. * settlement. A rejection is contained (the subagent seam's dispose() is
  419. * not supposed to reject, but a backend that does anyway must not break
  420. * quiescence): logged, and the child still leaves the registry.
  421. * @param callId - the child's registry key.
  422. * @param record - the registered child (the caller looked it up).
  423. * @returns resolves when the disposal settled either way; never rejects.
  424. */
  425. private disposeChild(callId: number, record: ChildRecord): Promise<void> {
  426. if (record.disposal !== undefined) return record.disposal
  427. record.disposal = Promise.resolve()
  428. .then(() => record.run.dispose())
  429. .catch((error: unknown) => {
  430. this.ctx.logger.warn(`workflow-worker-thread: child dispose failed: ${renderThrown(error)}`)
  431. })
  432. .then(() => { this.finishChild(callId) })
  433. return record.disposal
  434. }
  435. /** Drop a child record and release quiescence waiters when all work ends. */
  436. private finishChild(callId: number): void {
  437. this.children.delete(callId)
  438. this.notifyChildQuiescence()
  439. }
  440. /** Retire one provider startup transaction. */
  441. private finishPendingStart(task: Promise<void>): void {
  442. this.pendingStarts.delete(task)
  443. this.notifyChildQuiescence()
  444. }
  445. /** Release waiters only after both pending starts and published children end. */
  446. private notifyChildQuiescence(): void {
  447. if (this.children.size !== 0 || this.pendingStarts.size !== 0) return
  448. for (const waiter of this.quiescenceWaiters.splice(0)) waiter()
  449. }
  450. /** Resolves once every pending start and published child has reached quiescence. */
  451. private childQuiescence(): Promise<void> {
  452. if (this.children.size === 0 && this.pendingStarts.size === 0) return Promise.resolve()
  453. return new Promise((resolve) => { this.quiescenceWaiters.push(resolve) })
  454. }
  455. /** Abort + dispose every registered child (worker death / final teardown); disposal is contained, not awaited. */
  456. private reapChildren(reason: string): void {
  457. this.abortChildren(this.cancelReason ?? reason)
  458. for (const [callId, record] of [...this.children]) {
  459. void this.disposeChild(callId, record)
  460. }
  461. }
  462. /** Abort the one canonical signal shared by pending and published children. */
  463. private abortChildren(reason: string): void {
  464. if (!this.controller.signal.aborted) this.controller.abort(reason)
  465. }
  466. private onResult(result: WorkflowResult): void {
  467. // The owned worker session sends one Result. Keep a late duplicate or a
  468. // Result queued behind another terminal source completely side-effect-free.
  469. if (this.terminalClaimed) return
  470. // First-wins is decided when the Result message reaches the host. If no
  471. // external cancellation was already in flight, this result won. Reaping a
  472. // stray child below may synchronously reenter cancel() through provider
  473. // callbacks, but that internal post-result cleanup must not retroactively
  474. // rewrite the worker result that arrived first.
  475. const cancellationWasRequested = this.cancelReason !== undefined
  476. // Claim before settlement cleanup invokes provider disposal. Once Result
  477. // won, a later cancellation cannot rewrite it.
  478. this.terminalClaimed = true
  479. // Abort pending starts and begin disposing published children before the
  480. // workflow becomes externally settled. Cleanup remains independently
  481. // tracked by childQuiescence and the holder's dispose().
  482. this.reapChildren('workflow settled')
  483. if (!cancellationWasRequested) {
  484. this.settleResult(result)
  485. return
  486. }
  487. if (result.stopReason !== 'cancelled') {
  488. // The script settled while our cancel was crossing the thread boundary
  489. // — the seam-visible result had NOT settled when cancellation was
  490. // requested, so report cancelled (the vm drive()'s post-settle check,
  491. // relocated to the receiving side of the race).
  492. this.settleResult(this.cancelledResult(result.agentsStarted))
  493. return
  494. }
  495. this.settleResult(result)
  496. }
  497. /** Process an error/messageerror/exit signal; `exit` also performs the final disposal sweep. */
  498. private onWorkerDeath(message: string, isExit: boolean): void {
  499. if (!this.workerDeathObserved) {
  500. // Close message admission BEFORE cleanup callbacks: Node can deliver a
  501. // message queued before the crash after its `error` event. Treating the
  502. // first death signal as a logical barrier prevents that late message
  503. // from creating work or narrating after workflow/end.
  504. this.workerDeathObserved = true
  505. const outcomeWasClaimed = this.terminalClaimed
  506. const cancellationWasRequested = this.cancelReason !== undefined
  507. // When death is itself the terminal source, claim BEFORE child reap or
  508. // synthesized observer callbacks. Either can reenter cancel(); a death
  509. // that arrived first remains an error, while a cancellation already
  510. // accepted before death remains cancelled. If Result/grace already won,
  511. // preserve it while still performing prompt failure-time cleanup.
  512. if (!outcomeWasClaimed) this.terminalClaimed = true
  513. if (this.children.size > 0 || this.pendingStarts.size > 0) this.reapChildren('workflow worker gone')
  514. this.endStrandedAgents()
  515. if (!outcomeWasClaimed) {
  516. if (cancellationWasRequested) {
  517. this.settleResult(this.cancelledResult(this.hostStarted))
  518. } else {
  519. this.settleResult({ value: null, stopReason: 'error', error: message, agentsStarted: this.hostStarted })
  520. }
  521. }
  522. }
  523. if (!isExit) return
  524. // `error` is not Node's physical delivery barrier: a queued message may
  525. // precede `exit`. Admission is already closed, so this final sweep only
  526. // joins/starts disposal for registry survivors; it deliberately does not
  527. // repeat explicit provider cancellation.
  528. for (const [callId, record] of [...this.children]) void this.disposeChild(callId, record)
  529. this.endStrandedAgents()
  530. }
  531. /**
  532. * The single agent-end emission gate: forwards `end` iff its start is still
  533. * unpaired in the ledger, so every forwarded `workflow/agent-start` gets
  534. * EXACTLY one `workflow/agent-end` — the worker's own report where it can
  535. * speak, a host-synthesized one where it cannot ({@link endStrandedAgents}).
  536. * @param end - the settlement to emit (worker-reported or synthesized).
  537. */
  538. private endAgent(end: WorkflowAgentEndInfo): void {
  539. /* v8 ignore next -- a real end still in flight across the grace force-settle: not orderable in-process */
  540. if (!this.liveAgents.delete(end.seq)) return
  541. this.observer.agentEnd(end)
  542. }
  543. /**
  544. * Synthesize the missing `agent-end` for every started-but-unpaired agent,
  545. * outcome `'cancelled'`: the reap cancels every child, and a real
  546. * settlement racing the force-settle loses to that already-started external
  547. * cancellation. The atomic terminal boundaries in {@link onResult} and
  548. * {@link onWorkerDeath} deliberately exclude teardown callbacks as contenders.
  549. * Called where the worker can no longer speak (the grace force-settle,
  550. * worker death, physical exit). When grace/death is the terminal source it
  551. * runs before settleResult, so already-known pairs precede `workflow/end`;
  552. * after an earlier Result, exit cleanup may close a survivor afterward.
  553. * The ledger preserves exactly-once pairing in both orders.
  554. */
  555. private endStrandedAgents(): void {
  556. for (const info of [...this.liveAgents.values()]) {
  557. this.endAgent({ ...info, outcome: 'cancelled' })
  558. }
  559. }
  560. private cancelledResult(agentsStarted: number): WorkflowResult {
  561. // cancel() is the only writer of cancelReason and every caller checks it
  562. // first; the fallback guards the type, not a reachable path.
  563. /* v8 ignore next */
  564. const reason = this.cancelReason ?? 'workflow cancelled'
  565. return { value: null, stopReason: 'cancelled', error: `workflow run cancelled: ${reason}`, agentsStarted }
  566. }
  567. /** Remove the exact abort callback installed on the caller's start signal. */
  568. private detachInputSignal(): void {
  569. const signal = this.inputSignal
  570. const onAbort = this.inputSignalAbort
  571. if (signal === undefined || onAbort === undefined) return
  572. this.inputSignal = undefined
  573. this.inputSignalAbort = undefined
  574. signal.removeEventListener('abort', onAbort)
  575. }
  576. /** First settle wins; disarms the grace timer and releases the caller signal. */
  577. private settleResult(result: WorkflowResult): void {
  578. // Every current terminal source claims ownership before calling here; keep
  579. // the fallback local so a future caller cannot resolve twice.
  580. /* v8 ignore next -- defensive fallback outside the claimed state machine */
  581. if (this.settled) return
  582. this.terminalClaimed = true
  583. this.settled = true
  584. this.detachInputSignal()
  585. clearTimeout(this.graceTimer)
  586. this.settleResolve(result)
  587. }
  588. }
  589. /** A plain timer sleep (the dispose grace); unref'd so it never holds the process open. */
  590. function sleep(ms: number): Promise<void> {
  591. return new Promise((resolve) => {
  592. const timer = setTimeout(resolve, ms)
  593. timer.unref()
  594. })
  595. }