1
0

host.ts 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619
  1. /**
  2. * Host side of one workflow run. The first worker result, unexpected death, or
  3. * cancellation-grace expiry owns settlement and closes message admission.
  4. * Pending starts share one abort signal; published children share idempotent
  5. * cleanup, and quiescence waits for both while synthesizing any missing end events.
  6. * @module @deepseek-ai/dsh-workflow-worker-thread/host
  7. */
  8. import { tmpdir } from 'node:os'
  9. import { Worker } from 'node:worker_threads'
  10. import type { WorkerOptions } from 'node:worker_threads'
  11. import { fileURLToPath } from 'node:url'
  12. import type { Context } from '@deepseek-ai/cordis'
  13. import type { Agent } from '@deepseek-ai/dsh-agent'
  14. import { assertNever } from '@deepseek-ai/dsh-llm'
  15. import { snapshotJsonValue } from '@deepseek-ai/dsh-session'
  16. import type SubagentRuntime from '@deepseek-ai/dsh-subagent'
  17. import type { SubagentRun } from '@deepseek-ai/dsh-subagent'
  18. import type { WorkflowAgentEndInfo, WorkflowAgentInfo, WorkflowMeta, WorkflowResult, WorkflowRun, WorkflowRunId } from '@deepseek-ai/dsh-workflow'
  19. import { renderThrown } from './realm.ts'
  20. import type { ExecutionObserver } from './runtime.ts'
  21. import { HostToWorkerType, WorkerToHostType } from './protocol.ts'
  22. import type { HostToWorkerPayloads, WorkerToHostMessage } from './protocol.ts'
  23. import type { ChildResult, ChildStartRequest, WorkerInit } from './types.ts'
  24. /** One published child and its shared quiescent-disposal transaction. */
  25. interface ChildRecord {
  26. readonly run: SubagentRun
  27. disposal?: Promise<void>
  28. }
  29. /**
  30. * The scrubbed worker environment: no ambient credentials, no loader flags.
  31. * Windows derives `os.tmpdir()` from `TMP`/`TEMP` and falls back to the
  32. * literal relative path `undefined\temp` when the environment is empty, so
  33. * tsx's transform cache would land in a cwd-relative `undefined/temp`
  34. * directory; the host's real temp path (not a credential) is injected there.
  35. * The unbuilt shape additionally forwards `TSX_TSCONFIG_PATH` for path
  36. * resolution.
  37. * @param platform - host platform; overridable so tests exercise both peer arms.
  38. * @returns the scrubbed worker environment object.
  39. */
  40. export function workerSpawnEnv(platform: NodeJS.Platform = process.platform): NodeJS.ProcessEnv {
  41. const env: NodeJS.ProcessEnv = {}
  42. if (platform === 'win32') {
  43. const tmp = tmpdir()
  44. env.TMP = tmp
  45. env.TEMP = tmp
  46. }
  47. if (process.env.TSX_TSCONFIG_PATH !== undefined) {
  48. env.TSX_TSCONFIG_PATH = process.env.TSX_TSCONFIG_PATH
  49. }
  50. return env
  51. }
  52. /**
  53. * Resolve a built worker bundle or an unbuilt bootstrap that installs both tsx
  54. * transforms inside the worker. Both shapes clear `execArgv` and the ambient
  55. * environment (the worker only sees the platform temp path and, unbuilt,
  56. * `TSX_TSCONFIG_PATH`).
  57. * @param init - the run payload, passed as `workerData`.
  58. * @returns the entry path or URL and the Worker options to spawn it with.
  59. */
  60. function resolveWorkerSpawn(init: WorkerInit): { entry: string | URL; options: WorkerOptions } {
  61. /* v8 ignore next 3 -- the built-output arm: tests always run unbuilt (src/); the built-worker e2e exercises this shape for real */
  62. if (!import.meta.url.endsWith('.ts')) {
  63. return { entry: fileURLToPath(new URL('./worker.cjs', import.meta.url)), options: { workerData: init, env: workerSpawnEnv(), execArgv: [] } }
  64. }
  65. // Resolve tsx only for unbuilt consumers and install it before importing TS.
  66. const workerEntry = new URL('./worker.ts', import.meta.url)
  67. const tsxEsmApiEntry = import.meta.resolve('tsx/esm/api')
  68. const tsxCjsApiEntry = import.meta.resolve('tsx/cjs/api')
  69. const bootstrap = [
  70. `import { register as registerEsm } from ${JSON.stringify(tsxEsmApiEntry)}`,
  71. `import { register as registerCjs } from ${JSON.stringify(tsxCjsApiEntry)}`,
  72. 'registerCjs()',
  73. 'registerEsm()',
  74. `await import(${JSON.stringify(workerEntry.href)})`,
  75. ].join('\n')
  76. return {
  77. entry: new URL(`data:text/javascript,${encodeURIComponent(bootstrap)}`),
  78. options: {
  79. workerData: init,
  80. env: workerSpawnEnv(),
  81. execArgv: [],
  82. },
  83. }
  84. }
  85. /**
  86. * One live worker-engine run — the seam's {@link WorkflowRun}, returned by
  87. * `start()` directly. Owns the Worker, the child registry, and the result
  88. * settlement; `result` never rejects. `meta` is trusted same-process data
  89. * borrowed as immutable by the handle and lifecycle events. The holder-bound
  90. * SubagentRuntime handle is captured before the
  91. * engine returns this run, so unloading the engine removes only the ability to
  92. * start another workflow; this run can still start and clean up its children.
  93. */
  94. export class WorkerRun implements WorkflowRun {
  95. /** Settles exactly once with the run's outcome; never rejects. */
  96. readonly result: Promise<WorkflowResult>
  97. private settleResolve!: (result: WorkflowResult) => void
  98. private settled = false
  99. /** A Result/death/grace outcome atomically won before teardown callbacks. */
  100. private terminalClaimed = false
  101. /** The first death signal closes worker-message admission and owns failure-time cleanup. */
  102. private workerDeathObserved = false
  103. private cancelReason: string | undefined
  104. private graceTimer: NodeJS.Timeout | undefined
  105. private readonly worker: Worker
  106. /** Set on `exit`: the thread is gone, so posting has nowhere to go. */
  107. private workerGone = false
  108. /** Accepted `child-start` messages — the terminate-path `agentsStarted` (see module doc). */
  109. private hostStarted = 0
  110. /** Published children by callId; an entry leaves only after disposal settles. */
  111. private readonly children = new Map<number, ChildRecord>()
  112. /** Provider starts that have not yet fulfilled or rejected. */
  113. private readonly pendingStarts = new Set<Promise<void>>()
  114. /** Started-but-not-ended agents by seq — the pairing ledger the HOST guarantees (see {@link endAgent}). */
  115. private readonly liveAgents = new Map<number, WorkflowAgentInfo>()
  116. private readonly quiescenceWaiters: (() => void)[] = []
  117. /** The per-run abort fanout every child start request carries. */
  118. private readonly controller = new AbortController()
  119. /** External start signal and the exact callback installed on it, retained only until first settle/teardown. */
  120. private inputSignal: AbortSignal | undefined
  121. private inputSignalAbort: (() => void) | undefined
  122. private disposed: Promise<void> | undefined
  123. constructor(
  124. private readonly ctx: Context,
  125. private readonly subagents: SubagentRuntime,
  126. readonly id: WorkflowRunId,
  127. readonly meta: WorkflowMeta,
  128. private readonly parent: Agent,
  129. init: WorkerInit,
  130. private readonly provider: string,
  131. private readonly disposeGraceMs: number,
  132. private readonly observer: ExecutionObserver,
  133. signal: AbortSignal | undefined,
  134. ) {
  135. this.result = new Promise<WorkflowResult>((resolve) => { this.settleResolve = resolve })
  136. // workerData rides the structured clone: args are plain JSON by the seam
  137. // contract, so the clone is total and doubles as the caller-isolation
  138. // copy (a clone failure throws loud out of start()).
  139. const { entry, options } = resolveWorkerSpawn(init)
  140. this.worker = new Worker(entry, options)
  141. this.worker.on('message', (message: WorkerToHostMessage) => { this.onMessage(message) })
  142. this.worker.on('error', (error) => { this.onWorkerDeath(`workflow worker failed: ${renderThrown(error)}`, false) })
  143. /* v8 ignore next -- messageerror: not constructible from the engine's own protocol (every payload is JSON data) */
  144. this.worker.on('messageerror', (error) => { this.onWorkerDeath(`workflow worker message failed to deserialize: ${renderThrown(error)}`, false) })
  145. this.worker.on('exit', (code) => {
  146. this.workerGone = true
  147. this.onWorkerDeath(`workflow worker exited before the run settled (exit code ${code})`, true)
  148. })
  149. if (signal?.aborted) {
  150. this.cancel('workflow start signal already aborted')
  151. } else if (signal !== undefined) {
  152. const onAbort = (): void => {
  153. this.detachInputSignal()
  154. this.cancel('workflow signal aborted')
  155. }
  156. this.inputSignal = signal
  157. this.inputSignalAbort = onAbort
  158. signal.addEventListener('abort', onAbort, { once: true })
  159. }
  160. }
  161. /**
  162. * Cancel the run: the worker is told (its hooks start throwing and the
  163. * script dies at its next await), the required signal shared by every child
  164. * start is aborted, and the grace timer
  165. * arms: a run still unsettled `disposeGraceMs` later force-settles
  166. * `cancelled` and its worker is TERMINATED. Idempotent; the first reason
  167. * wins.
  168. * @param reason - human-readable cause (default `'workflow cancelled'`).
  169. */
  170. cancel(reason?: string): void {
  171. // A settled run has nothing left to cancel, and a terminal source claimed
  172. // before its cleanup callbacks must exclude cancellation reentered by one
  173. // of those callbacks. Without the settled guard the
  174. // ordinary consumer path (await result, then dispose -> cancel) would arm
  175. // a grace timer nothing ever clears, pinning the run and its Worker
  176. // closure until the grace expires - a bounded leak per completed run.
  177. if (this.settled || this.terminalClaimed || this.cancelReason !== undefined) return
  178. this.cancelReason = reason ?? 'workflow cancelled'
  179. this.post(HostToWorkerType.Cancel, { reason: this.cancelReason })
  180. this.abortChildren(this.cancelReason)
  181. this.graceTimer = setTimeout(() => {
  182. // Cancellation already owns the race through cancelReason; close the
  183. // terminal boundary explicitly before observer teardown callbacks.
  184. this.terminalClaimed = true
  185. // The worker may no longer speak (it is about to be terminated): pair
  186. // every stranded start before the run settles, so ends precede
  187. // workflow/end.
  188. this.endStrandedAgents()
  189. this.settleResult(this.cancelledResult(this.hostStarted))
  190. void this.worker.terminate()
  191. }, this.disposeGraceMs)
  192. // unref'd: an armed grace timer must never hold the process open.
  193. this.graceTimer.unref()
  194. }
  195. /**
  196. * Cancel + bounded settle + termination. Host-drives every registered
  197. * child's disposal IMMEDIATELY — a wedged worker can relay no dispose RPC,
  198. * and deferring child teardown to the post-terminate reap would spend the
  199. * whole grace waiting for a quiescence that cannot start, then return with
  200. * the disposals still in flight — so child disposal overlaps the same
  201. * grace the worker gets to settle (the worker's own dispose RPCs join the
  202. * shared per-child disposal). Waits (at most the grace) for the result and
  203. * child quiescence, then terminates the worker unconditionally — the
  204. * thread never outlives its run — and reaps whatever children remain
  205. * (their disposal is contained, not awaited past the grace, the same
  206. * abandonment the seam documents for a slow-disposing child). Idempotent;
  207. * safe on every path.
  208. * @returns resolves when the run's resources are released or abandoned.
  209. */
  210. dispose(): Promise<void> {
  211. if (this.disposed !== undefined) return this.disposed
  212. // Claim the public transaction BEFORE its body invokes child/provider
  213. // disposal. A raw provider callback can reenter handle.dispose(); it must
  214. // join this promise rather than start a second traversal.
  215. const claimed = Promise.withResolvers<undefined>()
  216. this.disposed = claimed.promise
  217. void (async () => {
  218. this.detachInputSignal()
  219. this.cancel('workflow disposed')
  220. // cancel() deliberately becomes a no-op after terminal settlement, but
  221. // disposal still owns every registered child. Reap independently so an
  222. // already-settled workflow cannot wait on child quiescence before it has
  223. // started the surviving children's disposals. On an unsettled run this
  224. // joins the cancel path through the per-call cancellation/disposal gates.
  225. this.reapChildren('workflow disposed')
  226. await Promise.race([
  227. (async () => {
  228. await this.result
  229. await this.childQuiescence()
  230. })(),
  231. sleep(this.disposeGraceMs),
  232. ])
  233. await this.worker.terminate()
  234. this.reapChildren('workflow disposed')
  235. })().then(
  236. () => { claimed.resolve(undefined) },
  237. /* v8 ignore next -- result/quiescence never reject and Worker.terminate is the only external promise */
  238. (error: unknown) => { claimed.reject(error) },
  239. )
  240. return this.disposed
  241. }
  242. /** Post one message to the worker (payload looked up from the tag's map entry), tolerating a thread that is already gone. */
  243. private post<T extends HostToWorkerType>(type: T, payload: HostToWorkerPayloads[T]): void {
  244. if (this.workerGone || this.workerDeathObserved) return
  245. try {
  246. this.worker.postMessage({ type, ...payload })
  247. } catch (error: unknown) {
  248. // Only a teardown race can land here (every engine message is JSON
  249. // data, so serialization cannot fail); there is nothing left to
  250. // deliver to — log and move on.
  251. /* v8 ignore next -- postMessage teardown race (a throw between exit and its event): not constructible in-process */
  252. this.ctx.logger.warn(`workflow-worker-thread: postMessage failed: ${renderThrown(error)}`)
  253. }
  254. }
  255. private onMessage(message: WorkerToHostMessage): void {
  256. // Node may emit `error`, then deliver an already-queued `message`, then
  257. // emit `exit`. The first death signal is the host's logical delivery
  258. // barrier: nothing arriving afterward may create a child, narrate after
  259. // workflow/end, or compete with the chosen outcome.
  260. if (this.workerDeathObserved) return
  261. switch (message.type) {
  262. case WorkerToHostType.Ready:
  263. this.post(HostToWorkerType.Go, {})
  264. break
  265. case WorkerToHostType.Phase:
  266. // Post-cancel narration is suppressed host-side: worker-side the
  267. // hooks throw once the cancel message is PROCESSED, but narration
  268. // already in flight (or emitted while the cancel crossed the
  269. // boundary) must not reach observers — nothing is emitted after
  270. // cancel() returns.
  271. if (this.cancelReason === undefined) this.observer.phase(message.title)
  272. break
  273. case WorkerToHostType.Log:
  274. if (this.cancelReason === undefined) this.observer.log(message.message)
  275. break
  276. case WorkerToHostType.AgentStart:
  277. this.liveAgents.set(message.info.seq, message.info)
  278. this.observer.agentStart(message.info)
  279. break
  280. case WorkerToHostType.AgentEnd:
  281. // NOT suppressed on cancel: cancelled children report their paired
  282. // agent-end with outcome 'cancelled'. The gate (with the termination
  283. // paths' synthesis) is what makes the one-pair-per-started-child
  284. // contract hold on every stop path.
  285. this.endAgent(message.info)
  286. break
  287. case WorkerToHostType.ChildStart:
  288. this.onChildStart(message.callId, message.request)
  289. break
  290. case WorkerToHostType.ChildDispose:
  291. this.onChildDispose(message.callId)
  292. break
  293. case WorkerToHostType.Result:
  294. this.onResult(message.result)
  295. break
  296. /* v8 ignore next 2 -- closed engine-owned union; the arm only makes adding a message type a compile error */
  297. default:
  298. assertNever(message, 'worker-to-host message')
  299. }
  300. }
  301. /** Why a ready provider result may no longer be admitted to the worker. */
  302. private childAdmissionFailure(): { reason: string; rendered: string } | undefined {
  303. if (this.cancelReason !== undefined) {
  304. return { reason: this.cancelReason, rendered: `workflow run cancelled: ${this.cancelReason}` }
  305. }
  306. if (this.workerDeathObserved) {
  307. return { reason: 'workflow worker gone', rendered: 'workflow worker is no longer available' }
  308. }
  309. if (this.terminalClaimed) {
  310. return { reason: 'workflow settled', rendered: 'workflow run already settled' }
  311. }
  312. return undefined
  313. }
  314. private onChildStart(callId: number, request: ChildStartRequest): void {
  315. const initialFailure = this.childAdmissionFailure()
  316. if (initialFailure !== undefined) {
  317. // Refuse after a terminal boundary: a child must never start on an
  318. // already-aborted signal (a provider subscribing only to future abort
  319. // events would never observe it).
  320. this.post(HostToWorkerType.ChildStartError, { callId, rendered: initialFailure.rendered })
  321. return
  322. }
  323. this.hostStarted += 1
  324. const task = this.startChild(callId, request)
  325. this.pendingStarts.add(task)
  326. void task.then(
  327. () => { this.finishPendingStart(task) },
  328. /* v8 ignore next -- startChild contains provider and cleanup failures */
  329. () => { this.finishPendingStart(task) },
  330. )
  331. }
  332. /** Await one provider-owned startup transaction and publish only while admitted. */
  333. private async startChild(callId: number, request: ChildStartRequest): Promise<void> {
  334. let run: SubagentRun
  335. try {
  336. run = await this.subagents.start(this.provider, {
  337. prompt: [{ type: 'text', text: request.prompt }],
  338. parent: this.parent,
  339. signal: this.controller.signal,
  340. ...request.schema !== undefined ? { outputSchema: request.schema } : {},
  341. ...request.provider !== undefined || request.model !== undefined
  342. ? {
  343. agentOptions: {
  344. ...request.provider !== undefined ? { provider: request.provider } : {},
  345. ...request.model !== undefined ? { model: request.model } : {},
  346. },
  347. }
  348. : {},
  349. })
  350. } catch (error: unknown) {
  351. const failure = this.childAdmissionFailure()
  352. this.post(HostToWorkerType.ChildStartError, {
  353. callId,
  354. rendered: failure?.rendered ?? renderThrown(error),
  355. })
  356. return
  357. }
  358. const failure = this.childAdmissionFailure()
  359. if (failure !== undefined) {
  360. this.post(HostToWorkerType.ChildStartError, { callId, rendered: failure.rendered })
  361. try {
  362. await run.dispose()
  363. } catch (error: unknown) {
  364. this.ctx.logger.warn(`workflow-worker-thread: refused child dispose failed: ${renderThrown(error)}`)
  365. }
  366. return
  367. }
  368. const record: ChildRecord = { run }
  369. this.children.set(callId, record)
  370. // Attach result forwarding before publishing the child handle. Because the
  371. // callback itself runs in a later microtask, ChildStarted is still posted
  372. // first even for an already-settled scripted provider.
  373. const forwardResult = run.result.then<() => void, () => void>(
  374. (result) => {
  375. try {
  376. const snapshot = snapshotJsonValue<ChildResult>({
  377. output: result.output,
  378. ...result.structured !== undefined ? { structured: result.structured } : {},
  379. stopReason: result.stopReason,
  380. })
  381. if (snapshot === undefined) throw new TypeError('child result is not losslessly JSON-serializable')
  382. return () => { this.post(HostToWorkerType.ChildSettled, { callId, result: snapshot }) }
  383. } catch (error: unknown) {
  384. const rendered = `workflow child result could not cross the worker boundary: ${renderThrown(error)}`
  385. return () => { this.post(HostToWorkerType.ChildFailed, { callId, rendered }) }
  386. }
  387. },
  388. (error: unknown) => {
  389. const rendered = renderThrown(error)
  390. return () => { this.post(HostToWorkerType.ChildFailed, { callId, rendered }) }
  391. },
  392. )
  393. this.post(HostToWorkerType.ChildStarted, { callId, childId: run.id })
  394. void forwardResult.then((forward) => { forward() })
  395. }
  396. private onChildDispose(callId: number): void {
  397. const record = this.children.get(callId)
  398. if (record === undefined) {
  399. // Already disposed host-side (a dispose() drive or a death reap beat
  400. // the RPC) — the ack is still owed (the worker-side wrapper awaits it).
  401. this.post(HostToWorkerType.ChildDisposed, { callId })
  402. return
  403. }
  404. // disposeChild never rejects (containment is inside), so the ack always follows.
  405. void this.disposeChild(callId, record).then(() => { this.post(HostToWorkerType.ChildDisposed, { callId }) })
  406. }
  407. /**
  408. * Start (or join) one registered child's disposal; the registry entry
  409. * leaves when it settles. Memoized per callId: the worker's dispose RPC,
  410. * the dispose() host drive, and the reap can all land on the same child —
  411. * the child's `dispose()` runs once and every caller awaits that one
  412. * settlement. A rejection is contained (the subagent seam's dispose() is
  413. * not supposed to reject, but a backend that does anyway must not break
  414. * quiescence): logged, and the child still leaves the registry.
  415. * @param callId - the child's registry key.
  416. * @param record - the registered child (the caller looked it up).
  417. * @returns resolves when the disposal settled either way; never rejects.
  418. */
  419. private disposeChild(callId: number, record: ChildRecord): Promise<void> {
  420. if (record.disposal !== undefined) return record.disposal
  421. record.disposal = Promise.resolve()
  422. .then(() => record.run.dispose())
  423. .catch((error: unknown) => {
  424. this.ctx.logger.warn(`workflow-worker-thread: child dispose failed: ${renderThrown(error)}`)
  425. })
  426. .then(() => { this.finishChild(callId) })
  427. return record.disposal
  428. }
  429. /** Drop a child record and release quiescence waiters when all work ends. */
  430. private finishChild(callId: number): void {
  431. this.children.delete(callId)
  432. this.notifyChildQuiescence()
  433. }
  434. /** Retire one provider startup transaction. */
  435. private finishPendingStart(task: Promise<void>): void {
  436. this.pendingStarts.delete(task)
  437. this.notifyChildQuiescence()
  438. }
  439. /** Release waiters only after both pending starts and published children end. */
  440. private notifyChildQuiescence(): void {
  441. if (this.children.size !== 0 || this.pendingStarts.size !== 0) return
  442. for (const waiter of this.quiescenceWaiters.splice(0)) waiter()
  443. }
  444. /** Resolves once every pending start and published child has reached quiescence. */
  445. private childQuiescence(): Promise<void> {
  446. if (this.children.size === 0 && this.pendingStarts.size === 0) return Promise.resolve()
  447. return new Promise((resolve) => { this.quiescenceWaiters.push(resolve) })
  448. }
  449. /** Abort + dispose every registered child (worker death / final teardown); disposal is contained, not awaited. */
  450. private reapChildren(reason: string): void {
  451. this.abortChildren(this.cancelReason ?? reason)
  452. for (const [callId, record] of [...this.children]) {
  453. void this.disposeChild(callId, record)
  454. }
  455. }
  456. /** Abort the one canonical signal shared by pending and published children. */
  457. private abortChildren(reason: string): void {
  458. if (!this.controller.signal.aborted) this.controller.abort(reason)
  459. }
  460. private onResult(result: WorkflowResult): void {
  461. // The owned worker session sends one Result. Keep a late duplicate or a
  462. // Result queued behind another terminal source completely side-effect-free.
  463. if (this.terminalClaimed) return
  464. // First-wins is decided when the Result message reaches the host. If no
  465. // external cancellation was already in flight, this result won. Reaping a
  466. // stray child below may synchronously reenter cancel() through provider
  467. // callbacks, but that internal post-result cleanup must not retroactively
  468. // rewrite the worker result that arrived first.
  469. const cancellationWasRequested = this.cancelReason !== undefined
  470. // Claim before settlement cleanup invokes provider disposal. Once Result
  471. // won, a later cancellation cannot rewrite it.
  472. this.terminalClaimed = true
  473. // Abort pending starts and begin disposing published children before the
  474. // workflow becomes externally settled. Cleanup remains independently
  475. // tracked by childQuiescence and the holder's dispose().
  476. this.reapChildren('workflow settled')
  477. if (!cancellationWasRequested) {
  478. this.settleResult(result)
  479. return
  480. }
  481. if (result.stopReason !== 'cancelled') {
  482. // The script settled while our cancel was crossing the thread boundary
  483. // — the seam-visible result had NOT settled when cancellation was
  484. // requested, so report cancelled (the vm drive()'s post-settle check,
  485. // relocated to the receiving side of the race).
  486. this.settleResult(this.cancelledResult(result.agentsStarted))
  487. return
  488. }
  489. this.settleResult(result)
  490. }
  491. /** Process an error/messageerror/exit signal; `exit` also performs the final disposal sweep. */
  492. private onWorkerDeath(message: string, isExit: boolean): void {
  493. if (!this.workerDeathObserved) {
  494. // Close message admission BEFORE cleanup callbacks: Node can deliver a
  495. // message queued before the crash after its `error` event. Treating the
  496. // first death signal as a logical barrier prevents that late message
  497. // from creating work or narrating after workflow/end.
  498. this.workerDeathObserved = true
  499. const outcomeWasClaimed = this.terminalClaimed
  500. const cancellationWasRequested = this.cancelReason !== undefined
  501. // When death is itself the terminal source, claim BEFORE child reap or
  502. // synthesized observer callbacks. Either can reenter cancel(); a death
  503. // that arrived first remains an error, while a cancellation already
  504. // accepted before death remains cancelled. If Result/grace already won,
  505. // preserve it while still performing prompt failure-time cleanup.
  506. if (!outcomeWasClaimed) this.terminalClaimed = true
  507. if (this.children.size > 0 || this.pendingStarts.size > 0) this.reapChildren('workflow worker gone')
  508. this.endStrandedAgents()
  509. if (!outcomeWasClaimed) {
  510. if (cancellationWasRequested) {
  511. this.settleResult(this.cancelledResult(this.hostStarted))
  512. } else {
  513. this.settleResult({ value: null, stopReason: 'error', error: message, agentsStarted: this.hostStarted })
  514. }
  515. }
  516. }
  517. if (!isExit) return
  518. // `error` is not Node's physical delivery barrier: a queued message may
  519. // precede `exit`. Admission is already closed, so this final sweep only
  520. // joins/starts disposal for registry survivors; it deliberately does not
  521. // repeat explicit provider cancellation.
  522. for (const [callId, record] of [...this.children]) void this.disposeChild(callId, record)
  523. this.endStrandedAgents()
  524. }
  525. /**
  526. * The single agent-end emission gate: forwards `end` iff its start is still
  527. * unpaired in the ledger, so every forwarded `workflow/agent-start` gets
  528. * EXACTLY one `workflow/agent-end` — the worker's own report where it can
  529. * speak, a host-synthesized one where it cannot ({@link endStrandedAgents}).
  530. * @param end - the settlement to emit (worker-reported or synthesized).
  531. */
  532. private endAgent(end: WorkflowAgentEndInfo): void {
  533. /* v8 ignore next -- a real end still in flight across the grace force-settle: not orderable in-process */
  534. if (!this.liveAgents.delete(end.seq)) return
  535. this.observer.agentEnd(end)
  536. }
  537. /**
  538. * Synthesize the missing `agent-end` for every started-but-unpaired agent,
  539. * outcome `'cancelled'`: the reap cancels every child, and a real
  540. * settlement racing the force-settle loses to that already-started external
  541. * cancellation. The atomic terminal boundaries in {@link onResult} and
  542. * {@link onWorkerDeath} deliberately exclude teardown callbacks as contenders.
  543. * Called where the worker can no longer speak (the grace force-settle,
  544. * worker death, physical exit). When grace/death is the terminal source it
  545. * runs before settleResult, so already-known pairs precede `workflow/end`;
  546. * after an earlier Result, exit cleanup may close a survivor afterward.
  547. * The ledger preserves exactly-once pairing in both orders.
  548. */
  549. private endStrandedAgents(): void {
  550. for (const info of [...this.liveAgents.values()]) {
  551. this.endAgent({ ...info, outcome: 'cancelled' })
  552. }
  553. }
  554. private cancelledResult(agentsStarted: number): WorkflowResult {
  555. // cancel() is the only writer of cancelReason and every caller checks it
  556. // first; the fallback guards the type, not a reachable path.
  557. /* v8 ignore next */
  558. const reason = this.cancelReason ?? 'workflow cancelled'
  559. return { value: null, stopReason: 'cancelled', error: `workflow run cancelled: ${reason}`, agentsStarted }
  560. }
  561. /** Remove the exact abort callback installed on the caller's start signal. */
  562. private detachInputSignal(): void {
  563. const signal = this.inputSignal
  564. const onAbort = this.inputSignalAbort
  565. if (signal === undefined || onAbort === undefined) return
  566. this.inputSignal = undefined
  567. this.inputSignalAbort = undefined
  568. signal.removeEventListener('abort', onAbort)
  569. }
  570. /** First settle wins; disarms the grace timer and releases the caller signal. */
  571. private settleResult(result: WorkflowResult): void {
  572. // Every current terminal source claims ownership before calling here; keep
  573. // the fallback local so a future caller cannot resolve twice.
  574. /* v8 ignore next -- defensive fallback outside the claimed state machine */
  575. if (this.settled) return
  576. this.terminalClaimed = true
  577. this.settled = true
  578. this.detachInputSignal()
  579. clearTimeout(this.graceTimer)
  580. this.settleResolve(result)
  581. }
  582. }
  583. /** A plain timer sleep (the dispose grace); unref'd so it never holds the process open. */
  584. function sleep(ms: number): Promise<void> {
  585. return new Promise((resolve) => {
  586. const timer = setTimeout(resolve, ms)
  587. timer.unref()
  588. })
  589. }