subprocess.spec.ts 71 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793
  1. import { once } from 'node:events'
  2. import { Context } from '@deepseek-ai/cordis'
  3. import {
  4. CommandExitError,
  5. FileNotFoundError,
  6. SandboxNotFoundError,
  7. type CommandHandle,
  8. type CommandResult,
  9. type Sandbox,
  10. } from '@deepseek-ai/dsh-e2b'
  11. import type E2BRuntime from '@deepseek-ai/dsh-e2b'
  12. import type { SubprocessSpawnSpec } from '@deepseek-ai/dsh-subprocess'
  13. import E2BSubprocessRuntime from '@deepseek-ai/dsh-subprocess-e2b'
  14. import * as E2BSubprocessInvariant from '../src/invariant.ts'
  15. import { E2BBase64Decoder, E2B_OUTPUT_COMPLETE_FRAME, E2BOutputReader } from '../src/output.ts'
  16. import { E2BSubprocessHandle } from '../src/process.ts'
  17. import InvariantRegistry from '@deepseek-ai/dsh-invariants'
  18. import { describe, expect, it, vi } from 'vitest'
  19. function commandError(exitCode: number): CommandExitError {
  20. return new CommandExitError({ exitCode, stdout: '', stderr: '', error: `exit ${exitCode}` })
  21. }
  22. interface StartOptions {
  23. background: true
  24. cwd: string
  25. stdin: boolean
  26. timeoutMs: number
  27. signal?: AbortSignal
  28. envs?: Record<string, string>
  29. onStdout?: (data: string) => void | Promise<void>
  30. onStderr?: (data: string) => void | Promise<void>
  31. }
  32. class FakeCommandHandle {
  33. pid = 4242
  34. readonly sent: Array<string | Uint8Array> = []
  35. closes = 0
  36. kills = 0
  37. disconnects = 0
  38. killError: unknown
  39. killResult = true
  40. disconnectError: unknown
  41. private readonly result = Promise.withResolvers<CommandResult>()
  42. private settled = false
  43. constructor(private readonly onKill: () => void = () => {}) {}
  44. wait(): Promise<CommandResult> {
  45. return this.result.promise
  46. }
  47. async sendStdin(data: string | Uint8Array): Promise<void> {
  48. this.sent.push(data)
  49. }
  50. async closeStdin(): Promise<void> {
  51. this.closes += 1
  52. }
  53. async kill(): Promise<boolean> {
  54. this.kills += 1
  55. if (this.killError !== undefined) throw this.killError
  56. this.onKill()
  57. return this.killResult
  58. }
  59. async disconnect(): Promise<void> {
  60. this.disconnects += 1
  61. if (this.disconnectError !== undefined) throw this.disconnectError
  62. }
  63. succeed(exitCode = 0): void {
  64. if (this.settled) return
  65. this.settled = true
  66. this.result.resolve({ exitCode, stdout: '', stderr: '' })
  67. }
  68. fail(exitCode: number): void {
  69. if (this.settled) return
  70. this.settled = true
  71. this.result.reject(commandError(exitCode))
  72. }
  73. crash(error: unknown): void {
  74. if (this.settled) return
  75. this.settled = true
  76. this.result.reject(error)
  77. }
  78. }
  79. class FakeSandbox {
  80. readonly handle: FakeCommandHandle
  81. readonly commandsSeen: string[] = []
  82. readonly writtenFiles: string[][] = []
  83. readonly writtenFileData = new Map<string, string>()
  84. readonly removed: string[] = []
  85. readonly directories: string[] = []
  86. startOptions: StartOptions | undefined
  87. backgroundError: unknown
  88. envError: unknown
  89. statusError: unknown
  90. nextRemoveError: unknown
  91. probeError: unknown
  92. signalError: unknown
  93. readonly signalErrors: unknown[] = []
  94. trapsTerm = false
  95. delaysKill = false
  96. delaysKillCompletion = false
  97. sdkKillStops = true
  98. alive = true
  99. zombieOnly = false
  100. ambient = 'PATH=/ambient/bin\0KEEP=safe\0UNICODE=你好\0NPM_TOKEN=secret\0DSH_STALE=old\0BROKEN\0=bad\0'
  101. environmentHome = '/home/user'
  102. environmentWire: string | undefined
  103. environmentRequest: ((signal: AbortSignal | undefined) => Promise<void>) | undefined
  104. processGroupId = '4242\n'
  105. exitStatus = ''
  106. statusReads = 0
  107. readonly processGroupReads: string[] = []
  108. afterStatusRead: (() => void) | undefined
  109. beforeProbe: (() => void) | undefined
  110. afterProbe: (() => void) | undefined
  111. private startGate: Promise<void> | undefined
  112. private openStart: (() => void) | undefined
  113. private processGroupReadGate: Promise<void> | undefined
  114. private openProcessGroupRead: (() => void) | undefined
  115. private signalGate: Promise<void> | undefined
  116. private openSignal: (() => void) | undefined
  117. constructor() {
  118. this.handle = new FakeCommandHandle(() => {
  119. if (this.sdkKillStops) {
  120. this.alive = false
  121. this.handle.fail(137)
  122. }
  123. })
  124. }
  125. deferStart(): void {
  126. const gate = Promise.withResolvers<undefined>()
  127. this.startGate = gate.promise
  128. this.openStart = () => { gate.resolve(undefined) }
  129. }
  130. releaseStart(): void {
  131. this.openStart?.()
  132. }
  133. deferProcessGroupRead(): void {
  134. const gate = Promise.withResolvers<undefined>()
  135. this.processGroupReadGate = gate.promise
  136. this.openProcessGroupRead = () => { gate.resolve(undefined) }
  137. }
  138. releaseProcessGroupRead(): void {
  139. this.openProcessGroupRead?.()
  140. }
  141. deferSignals(): void {
  142. const gate = Promise.withResolvers<undefined>()
  143. this.signalGate = gate.promise
  144. this.openSignal = () => { gate.resolve(undefined) }
  145. }
  146. releaseSignals(): void {
  147. this.openSignal?.()
  148. }
  149. finish(exitCode = 0): void {
  150. this.alive = false
  151. void this.completeOutput().then(
  152. () => {
  153. if (exitCode === 0) this.handle.succeed(0)
  154. else this.handle.fail(exitCode)
  155. },
  156. (error: unknown) => { this.handle.crash(error) },
  157. )
  158. }
  159. async completeOutput(): Promise<void> {
  160. await Promise.all([
  161. this.stdoutWire(`${E2B_OUTPUT_COMPLETE_FRAME}\n`),
  162. this.stderrWire(`${E2B_OUTPUT_COMPLETE_FRAME}\n`),
  163. ])
  164. }
  165. async stdout(data: string): Promise<void> {
  166. await this.stdoutWire(data.length === 0 ? '' : `${Buffer.from(data).toString('base64')}\n`)
  167. }
  168. async stderr(data: string): Promise<void> {
  169. await this.stderrWire(data.length === 0 ? '' : `${Buffer.from(data).toString('base64')}\n`)
  170. }
  171. async stdoutWire(data: string): Promise<void> {
  172. await this.startOptions?.onStdout?.(data)
  173. }
  174. async stderrWire(data: string): Promise<void> {
  175. await this.startOptions?.onStderr?.(data)
  176. }
  177. readonly sandbox = {
  178. sandboxId: 'fake',
  179. files: {
  180. makeDir: async (path: string): Promise<boolean> => {
  181. this.directories.push(path)
  182. return true
  183. },
  184. write: async (files: Array<{ path: string; data: string }>): Promise<object[]> => {
  185. this.writtenFiles.push(files.map(file => file.path))
  186. for (const file of files) this.writtenFileData.set(file.path, file.data)
  187. return files.map(() => ({}))
  188. },
  189. read: async (path: string): Promise<string> => {
  190. if (!path.endsWith('/exit-code')) {
  191. await this.processGroupReadGate
  192. return this.processGroupReads.shift() ?? this.processGroupId
  193. }
  194. if (this.statusError !== undefined) {
  195. const error = this.statusError
  196. this.statusError = undefined
  197. throw error
  198. }
  199. this.statusReads += 1
  200. this.afterStatusRead?.()
  201. return this.exitStatus
  202. },
  203. remove: async (path: string): Promise<void> => {
  204. this.removed.push(path)
  205. if (this.nextRemoveError !== undefined) {
  206. const error = this.nextRemoveError
  207. this.nextRemoveError = undefined
  208. throw error
  209. }
  210. },
  211. },
  212. commands: {
  213. run: async (command: string, options?: StartOptions | { signal?: AbortSignal }): Promise<CommandHandle | CommandResult> => {
  214. this.commandsSeen.push(command)
  215. if (command.includes('env -0 | base64')) {
  216. await this.environmentRequest?.(options?.signal)
  217. if (this.envError !== undefined) throw this.envError
  218. return {
  219. exitCode: 0,
  220. stdout: this.environmentWire ?? [this.environmentHome, this.ambient]
  221. .map(value => Buffer.from(value).toString('base64'))
  222. .join('\n'),
  223. stderr: '',
  224. }
  225. }
  226. if (command.startsWith('set -o pipefail; ps -eo pgid=,stat=')) {
  227. this.beforeProbe?.()
  228. if (options?.signal?.aborted === true) throw new DOMException('aborted', 'AbortError')
  229. if (this.probeError !== undefined) {
  230. const error = this.probeError
  231. this.probeError = undefined
  232. throw error
  233. }
  234. const stdout = this.alive && !this.zombieOnly ? 'live\n' : ''
  235. this.afterProbe?.()
  236. return { exitCode: 0, stdout, stderr: '' }
  237. }
  238. if (command.startsWith('kill -TERM ')) {
  239. await this.signalGate
  240. const error = this.signalErrors.shift() ?? this.signalError
  241. if (error !== undefined) {
  242. if (this.signalErrors.length === 0) this.signalError = undefined
  243. throw error
  244. }
  245. if (!this.trapsTerm) {
  246. this.alive = false
  247. this.handle.fail(143)
  248. }
  249. return { exitCode: 0, stdout: '', stderr: '' }
  250. }
  251. if (command.startsWith('kill -KILL ')) {
  252. await this.signalGate
  253. const error = this.signalErrors.shift() ?? this.signalError
  254. if (error !== undefined) {
  255. if (this.signalErrors.length === 0) this.signalError = undefined
  256. throw error
  257. }
  258. if (!this.delaysKill) this.alive = false
  259. if (!this.delaysKillCompletion) this.handle.fail(137)
  260. return { exitCode: 0, stdout: '', stderr: '' }
  261. }
  262. if ((options as StartOptions | undefined)?.background === true) {
  263. this.startOptions = options as StartOptions
  264. await this.startGate
  265. if (this.backgroundError !== undefined) throw this.backgroundError
  266. return this.handle as unknown as CommandHandle
  267. }
  268. return { exitCode: 0, stdout: '', stderr: '' }
  269. },
  270. },
  271. } as unknown as Sandbox
  272. }
  273. function spec(overrides: Partial<SubprocessSpawnSpec> = {}): SubprocessSpawnSpec {
  274. return {
  275. argv: ['bash', '-c', 'printf ok'],
  276. cwd: '/workspace',
  277. stdio: {
  278. stdin: 'ignore',
  279. stdout: { maxBytes: 4, spill: { maxBytes: 16 } },
  280. stderr: { maxBytes: 4 },
  281. },
  282. graceMs: 5,
  283. ...overrides,
  284. }
  285. }
  286. function runtime(fake: FakeSandbox, getSandbox: () => Promise<Sandbox> = async () => fake.sandbox): E2BRuntime {
  287. return {
  288. cwd: '/workspace',
  289. runtimeRoot: '/workspace/.dsh-e2b',
  290. getSandbox,
  291. } as unknown as E2BRuntime
  292. }
  293. async function flush(): Promise<void> {
  294. await new Promise(resolve => setTimeout(resolve, 0))
  295. }
  296. /** Construct the handle under test with the config default the service would pass. */
  297. function testHandle(
  298. runtime: ConstructorParameters<typeof E2BSubprocessHandle>[0],
  299. spec: ConstructorParameters<typeof E2BSubprocessHandle>[1],
  300. stateDir: string,
  301. pollMs = 20,
  302. ): E2BSubprocessHandle {
  303. return new E2BSubprocessHandle(runtime, spec, stateDir, pollMs)
  304. }
  305. describe('E2BOutputReader', () => {
  306. it('decodes base64 across arbitrary callback boundaries and rejects malformed framing', () => {
  307. const decoder = new E2BBase64Decoder()
  308. expect(decoder.push('')).toEqual(Buffer.alloc(0))
  309. expect(decoder.push('5')).toEqual(Buffer.alloc(0))
  310. expect(decoder.push('L2')).toEqual(Buffer.alloc(0))
  311. expect(decoder.push('g\n').toString()).toBe('你')
  312. expect(decoder.push('YQ==\nYg==\n').toString()).toBe('ab')
  313. expect(decoder.push(`${Buffer.from([0, 255]).toString('base64')}\n`)).toEqual(Buffer.from([0, 255]))
  314. expect(decoder.push(`${E2B_OUTPUT_COMPLETE_FRAME}\n`)).toEqual(Buffer.alloc(0))
  315. decoder.finish()
  316. expect(() => new E2BBase64Decoder().push('%\n')).toThrow('invalid base64')
  317. expect(() => new E2BBase64Decoder().push('AB==\n')).toThrow('invalid base64')
  318. expect(() => decoder.push(`${E2B_OUTPUT_COMPLETE_FRAME}\n`)).toThrow('duplicate output transport completion')
  319. expect(() => decoder.push('YQ==\n')).toThrow('continued after completion')
  320. const truncated = new E2BBase64Decoder()
  321. truncated.push('YQ')
  322. expect(() => { truncated.finish() }).toThrow('truncated base64')
  323. expect(() => { new E2BBase64Decoder().finish() }).toThrow('incomplete output transport')
  324. const interrupted = new E2BBase64Decoder()
  325. interrupted.push('YQ')
  326. expect(() => { interrupted.finish(false) }).not.toThrow()
  327. })
  328. it('keeps a byte-exact tail with independent whole-stream cursors', () => {
  329. const reader = new E2BOutputReader(4, 10, '/remote/spill')
  330. reader.push(Buffer.alloc(0))
  331. reader.push(Buffer.from('ab'))
  332. reader.push(Buffer.from('cdef'))
  333. expect(reader.size).toBe(6)
  334. expect(reader.readFrom(0)).toEqual({ text: 'cdef', nextOffset: 6, lossy: true, spillPath: '/remote/spill' })
  335. expect(reader.readFrom(2)).toEqual({ text: 'cdef', nextOffset: 6, lossy: false })
  336. expect(reader.readFrom(5)).toEqual({ text: 'f', nextOffset: 6, lossy: false })
  337. expect(reader.readFrom(99)).toEqual({ text: '', nextOffset: 6, lossy: false })
  338. reader.invalidateSpill()
  339. expect(reader.readFrom(0)).toEqual({ text: 'cdef', nextOffset: 6, lossy: true })
  340. })
  341. it('drops whole head chunks and withholds absent or over-cap spills', () => {
  342. const withoutSpill = new E2BOutputReader(2, undefined, '/unused')
  343. withoutSpill.push(Buffer.from('ab'))
  344. withoutSpill.push(Buffer.from('cd'))
  345. expect(withoutSpill.readFrom(0)).toEqual({ text: 'cd', nextOffset: 4, lossy: true })
  346. const overCap = new E2BOutputReader(2, 3, '/too-small')
  347. overCap.push(Buffer.from('abcd'))
  348. expect(overCap.readFrom(0)).toEqual({ text: 'cd', nextOffset: 4, lossy: true })
  349. })
  350. })
  351. describe('E2BSubprocessHandle', () => {
  352. it('starts asynchronously, keeps secrets out of the command, and supports deferred piped stdin/output', async () => {
  353. const fake = new FakeSandbox()
  354. fake.processGroupId = '4343\n'
  355. fake.deferStart()
  356. const handle = testHandle(runtime(fake), spec({
  357. argv: ['tool', 'argument with spaces'],
  358. stdio: { stdin: 'pipe', stdout: 'pipe', stderr: { maxBytes: 8, spill: { maxBytes: 32 } } },
  359. env: {
  360. PATH: '/bin',
  361. 'FOO-BAR': 'hyphen-value',
  362. '--split-string': 'literal-value',
  363. DEEPSEEK_API_KEY: 'explicit-secret',
  364. DSH_MODE: 'test',
  365. // The seam's tombstone: an explicit undefined removes the ambient entry.
  366. KEEP: undefined,
  367. },
  368. }), '/workspace/.dsh-e2b/processes/one')
  369. expect(handle.pid).toBeUndefined()
  370. handle.stdin!.write('hello')
  371. handle.stdin!.end()
  372. fake.releaseStart()
  373. await flush()
  374. expect(handle.pid).toBeUndefined()
  375. expect(fake.handle.sent.map(value => String(value))).toEqual(['hello'])
  376. expect(fake.handle.closes).toBe(1)
  377. const controlEnvs = fake.startOptions?.envs
  378. expect(controlEnvs?.HOME).toMatch(/^\/\.dsh-e2b-control-/)
  379. expect(controlEnvs).toEqual({
  380. TERM: 'dumb',
  381. NPM_TOKEN: '',
  382. DSH_STALE: '',
  383. HOME: controlEnvs?.HOME,
  384. })
  385. const command = fake.commandsSeen.find(value => value.includes('exec "$dsh_e2b_env_bin" -i'))!
  386. expect(command).toContain('"$dsh_e2b_setsid" --wait -- "$dsh_e2b_bash" -c')
  387. expect(command).not.toContain('DEEPSEEK_API_KEY')
  388. expect(command).not.toContain('DSH_MODE')
  389. expect(command).not.toContain('FOO-BAR')
  390. expect(command).not.toContain('explicit-secret')
  391. expect(command).not.toContain('hyphen-value')
  392. expect(command).not.toContain('${!dsh_e2b_name}')
  393. const environmentProbe = fake.commandsSeen.find(value => value.includes('env -0 | base64'))
  394. expect(environmentProbe).toContain('getent passwd "$(id -u)"')
  395. expect(environmentProbe).toContain('test -n "$dsh_e2b_home" -a -d "$dsh_e2b_home"')
  396. expect(environmentProbe).not.toContain('"$PWD"')
  397. expect(command).toContain('mapfile -d')
  398. expect(command).toContain('dsh_e2b_node="$(command -v node)"')
  399. expect(command).toContain('"$dsh_e2b_env_bin" -i "$dsh_e2b_node" -e')
  400. expect(command).toContain('"$dsh_e2b_env_bin" -i -- "${dsh_e2b_env[@]}" "$@"')
  401. expect(command).toContain('exec "$dsh_e2b_env_bin" -i -- "${dsh_e2b_env[@]}"')
  402. expect(command).toContain('>&2 2>/dev/null')
  403. expect(command).not.toContain('2>/dev/null >&2')
  404. expect(command).toContain('base64')
  405. expect(fake.writtenFiles[0]).toEqual([
  406. '/workspace/.dsh-e2b/processes/one/pid',
  407. '/workspace/.dsh-e2b/processes/one/exit-code',
  408. '/workspace/.dsh-e2b/processes/one/environment',
  409. '/workspace/.dsh-e2b/processes/one/stderr.log',
  410. ])
  411. expect(fake.writtenFileData.get('/workspace/.dsh-e2b/processes/one/environment')).toBe(
  412. 'PATH=/bin\0UNICODE=你好\0HOME=/home/user\0FOO-BAR=hyphen-value\0--split-string=literal-value\0DEEPSEEK_API_KEY=explicit-secret\0DSH_MODE=test\0',
  413. )
  414. let piped = ''
  415. handle.stdout!.on('data', (chunk) => { piped += String(chunk) })
  416. await fake.stdout('pipe-data')
  417. await fake.stderr('err')
  418. fake.finish()
  419. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  420. expect(piped).toBe('pipe-data')
  421. expect(handle.collected.stderr!.readFrom(0)).toMatchObject({ text: 'err', lossy: false })
  422. expect(fake.removed).toContain('/workspace/.dsh-e2b/processes/one/stderr.log')
  423. await expect(handle.waitForExit()).resolves.toBe(true)
  424. })
  425. it('rejects an unrepresentable graceMs before any remote work', () => {
  426. const ctx = new Context()
  427. const service = Object.create(E2BSubprocessRuntime.prototype) as E2BSubprocessRuntime
  428. Reflect.set(service, 'disposing', false)
  429. Reflect.set(service, 'ctx', ctx)
  430. for (const graceMs of [0, -1, Number.NaN, Number.POSITIVE_INFINITY]) {
  431. expect(() => service.spawn(spec({ graceMs }))).toThrow('graceMs must be a positive finite number')
  432. void expect(service.spawnTerminal({
  433. argv: ['bash'], cwd: '/w', rows: 24, cols: 80, graceMs,
  434. })).rejects.toThrow('graceMs must be a positive finite number')
  435. }
  436. })
  437. it('rejects malformed environment entries before command start', async () => {
  438. for (const env of [{ 'BAD=NAME': 'x' }, { BAD: 'x\0INJECTED=1' }]) {
  439. const fake = new FakeSandbox()
  440. const handle = testHandle(runtime(fake), spec({ env }), '/runtime/invalid-environment')
  441. await expect(handle.done).rejects.toThrow('environment entries')
  442. expect(fake.startOptions).toBeUndefined()
  443. expect(fake.removed).toContain('/runtime/invalid-environment')
  444. }
  445. })
  446. it('preserves UTF-8 bytes when the ASCII transport is split across callbacks', async () => {
  447. const fake = new FakeSandbox()
  448. const handle = testHandle(runtime(fake), spec({
  449. stdio: { stdin: 'ignore', stdout: 'pipe', stderr: { maxBytes: 4 } },
  450. }), '/runtime/split-utf8')
  451. await flush()
  452. const chunks: Buffer[] = []
  453. handle.stdout!.on('data', (chunk: Buffer) => { chunks.push(chunk) })
  454. for (const character of `${Buffer.from('A你好B').toString('base64')}\n`) {
  455. await fake.stdoutWire(character)
  456. }
  457. fake.finish()
  458. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  459. expect(Buffer.concat(chunks).toString('utf8')).toBe('A你好B')
  460. })
  461. it('rejects malformed output transport without confusing it with a consumer sink failure', async () => {
  462. const fake = new FakeSandbox()
  463. const handle = testHandle(runtime(fake), spec(), '/runtime/malformed-output')
  464. await flush()
  465. await fake.stdoutWire('%\n')
  466. fake.finish()
  467. await expect(handle.done).rejects.toThrow('invalid base64 output transport')
  468. const stderrFake = new FakeSandbox()
  469. const stderrHandle = testHandle(runtime(stderrFake), spec(), '/runtime/malformed-stderr')
  470. await flush()
  471. await stderrFake.stderrWire('%\n')
  472. stderrFake.finish()
  473. await expect(stderrHandle.done).rejects.toThrow('invalid base64 output transport')
  474. })
  475. it('rejects a naturally completed command whose encoder omits its completion frame', async () => {
  476. const fake = new FakeSandbox()
  477. const handle = testHandle(runtime(fake), spec(), '/runtime/incomplete-output')
  478. await flush()
  479. fake.alive = false
  480. fake.handle.succeed(0)
  481. await expect(handle.done).rejects.toThrow('incomplete output transport')
  482. })
  483. it('bounds descendant-held output draining and withholds the incomplete spill', async () => {
  484. const fake = new FakeSandbox()
  485. const handle = testHandle(runtime(fake), spec({ graceMs: 5 }), '/runtime/drain-bound')
  486. await flush()
  487. await fake.stdout('leader-output')
  488. fake.exitStatus = '0\n'
  489. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  490. expect(fake.handle.disconnects).toBe(1)
  491. expect(handle.collected.stdout?.readFrom(0)).toEqual({
  492. text: 'tput',
  493. nextOffset: 13,
  494. lossy: true,
  495. })
  496. expect(fake.removed).toContain('/runtime/drain-bound/stdout.log')
  497. handle.terminate()
  498. await expect(handle.waitForExit()).resolves.toBe(true)
  499. })
  500. it('releases an inherited-output callback blocked on host backpressure at drain expiry', async () => {
  501. const fake = new FakeSandbox()
  502. const written: string[] = []
  503. const stdoutWrite = vi.spyOn(process.stdout, 'write').mockImplementation(((chunk: Uint8Array) => {
  504. written.push(Buffer.from(chunk).toString())
  505. return false
  506. }) as typeof process.stdout.write)
  507. try {
  508. const handle = testHandle(runtime(fake), spec({
  509. graceMs: 5,
  510. stdio: { stdin: 'ignore', stdout: 'inherit', stderr: { maxBytes: 4 } },
  511. }), '/runtime/inherit-backpressure')
  512. await flush()
  513. let callbackSettled = false
  514. const blocked = fake.stdout('blocked bytes').then(() => { callbackSettled = true })
  515. await flush()
  516. expect(callbackSettled).toBe(false)
  517. fake.exitStatus = '0\n'
  518. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  519. await blocked
  520. expect(callbackSettled).toBe(true)
  521. expect(written.join('')).toBe('blocked bytes')
  522. expect(fake.handle.disconnects).toBe(1)
  523. handle.terminate()
  524. await expect(handle.waitForExit()).resolves.toBe(true)
  525. } finally {
  526. stdoutWrite.mockRestore()
  527. }
  528. })
  529. it('waits for lossless raw-pipe output after the direct status is published', async () => {
  530. const fake = new FakeSandbox()
  531. const handle = testHandle(runtime(fake), spec({
  532. graceMs: 1,
  533. stdio: { stdin: 'ignore', stdout: 'pipe', stderr: { maxBytes: 4 } },
  534. }), '/runtime/pipe-drain')
  535. let output = ''
  536. handle.stdout!.on('data', (chunk) => { output += String(chunk) })
  537. await flush()
  538. fake.exitStatus = '0\n'
  539. let settled = false
  540. void handle.done.then(() => { settled = true })
  541. await new Promise(resolve => setTimeout(resolve, 50))
  542. expect(settled).toBe(false)
  543. expect(fake.handle.disconnects).toBe(0)
  544. expect(fake.statusReads).toBe(0)
  545. await fake.stdout('complete protocol frame')
  546. fake.finish()
  547. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  548. expect(output).toBe('complete protocol frame')
  549. expect(fake.statusReads).toBe(1)
  550. })
  551. it('accepts clean encoder completion inside the output-drain grace', async () => {
  552. const fake = new FakeSandbox()
  553. const handle = testHandle(runtime(fake), spec({ graceMs: 100 }), '/runtime/drain-complete')
  554. await flush()
  555. fake.exitStatus = '0\n'
  556. fake.afterStatusRead = () => {
  557. fake.afterStatusRead = undefined
  558. setTimeout(() => { fake.finish() }, 0)
  559. }
  560. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  561. expect(fake.handle.disconnects).toBe(0)
  562. })
  563. it('preserves a published exit code when requested termination outlives output draining', async () => {
  564. const fake = new FakeSandbox()
  565. fake.trapsTerm = true
  566. fake.delaysKill = true
  567. fake.delaysKillCompletion = true
  568. fake.sdkKillStops = false
  569. const handle = testHandle(runtime(fake), spec({ graceMs: 5 }), '/runtime/drain-signal')
  570. await flush()
  571. handle.terminate()
  572. await vi.waitFor(() => { expect(fake.commandsSeen).toContain('kill -KILL -- -4242') })
  573. fake.exitStatus = '0\n'
  574. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  575. expect(fake.handle.disconnects).toBe(1)
  576. fake.alive = false
  577. handle.terminate()
  578. await expect(handle.waitForExit()).resolves.toBe(true)
  579. })
  580. it('preserves a published nonzero exit code when termination settles the SDK inside the drain grace', async () => {
  581. const fake = new FakeSandbox()
  582. const handle = testHandle(runtime(fake), spec({ graceMs: 100 }), '/runtime/drain-signal-settled')
  583. await flush()
  584. fake.exitStatus = '7\n'
  585. fake.afterStatusRead = () => {
  586. fake.afterStatusRead = undefined
  587. handle.terminate()
  588. }
  589. await expect(handle.done).resolves.toEqual({ exitCode: 7, signal: null })
  590. await expect(handle.waitForExit()).resolves.toBe(true)
  591. })
  592. it('rejects an invalid direct-command exit status', async () => {
  593. const fake = new FakeSandbox()
  594. const handle = testHandle(runtime(fake), spec(), '/runtime/invalid-status')
  595. await flush()
  596. fake.exitStatus = '999\n'
  597. await expect(handle.done).rejects.toThrow('invalid exit code')
  598. handle.terminate()
  599. await expect(handle.waitForExit()).resolves.toBe(true)
  600. })
  601. it('rolls back a published process group before rejecting a monitoring failure', async () => {
  602. const fake = new FakeSandbox()
  603. fake.statusError = new Error('status transport failed')
  604. const handle = testHandle(runtime(fake), spec(), '/runtime/status-failure')
  605. await expect(handle.done).rejects.toThrow('status transport failed')
  606. expect(fake.commandsSeen).toContain('kill -TERM -- -4242')
  607. expect(fake.alive).toBe(false)
  608. await expect(handle.waitForExit()).resolves.toBe(true)
  609. const failed = new FakeSandbox()
  610. failed.statusError = new Error('status transport failed')
  611. failed.signalErrors.push(new Error('TERM transport failed'), new Error('KILL transport failed'))
  612. failed.handle.killError = new Error('SDK kill failed')
  613. const retained = testHandle(runtime(failed), spec({ graceMs: 1 }), '/runtime/status-cleanup-failure')
  614. await expect(retained.done).rejects.toThrow(
  615. 'command monitoring failed and process-group rollback did not reach quiescence',
  616. )
  617. expect(failed.alive).toBe(true)
  618. failed.handle.killError = undefined
  619. retained.terminate()
  620. await expect(retained.waitForExit()).resolves.toBe(true)
  621. // A state-cleanup failure on top preserves the rollback failure instead of
  622. // re-aggregating only the original monitoring error.
  623. const triple = new FakeSandbox()
  624. triple.statusError = new Error('status transport failed')
  625. triple.signalErrors.push(new Error('TERM transport failed'), new Error('KILL transport failed'))
  626. triple.handle.killError = new Error('SDK kill failed')
  627. triple.nextRemoveError = new Error('state cleanup failed')
  628. const tripleHandle = testHandle(runtime(triple), spec({ graceMs: 1 }), '/runtime/triple-failure')
  629. const failure = await tripleHandle.done.catch((error: unknown) => error as AggregateError)
  630. expect(failure).toBeInstanceOf(AggregateError)
  631. expect((failure as AggregateError).message).toContain('private state cleanup failed')
  632. const nested = (failure as AggregateError).errors[0] as AggregateError
  633. expect(nested.message).toContain('rollback did not reach quiescence')
  634. triple.handle.killError = undefined
  635. tripleHandle.terminate()
  636. await expect(tripleHandle.waitForExit()).resolves.toBe(true)
  637. })
  638. it('surfaces deferred piped-stdin write and close failures as stream errors', async () => {
  639. const writeFake = new FakeSandbox()
  640. writeFake.deferStart()
  641. vi.spyOn(writeFake.handle, 'sendStdin').mockRejectedValueOnce('stdin rejected')
  642. const writeHandle = testHandle(runtime(writeFake), spec({
  643. stdio: { stdin: 'pipe', stdout: { maxBytes: 4 }, stderr: { maxBytes: 4 } },
  644. }), '/runtime/stdin-write-error')
  645. const writeError = once(writeHandle.stdin!, 'error')
  646. writeHandle.stdin!.write('input')
  647. writeFake.releaseStart()
  648. await expect(writeError).resolves.toMatchObject([{ message: 'stdin rejected' }])
  649. writeFake.finish()
  650. await writeHandle.done
  651. const closeFake = new FakeSandbox()
  652. vi.spyOn(closeFake.handle, 'closeStdin').mockRejectedValueOnce(new Error('close rejected'))
  653. const closeHandle = testHandle(runtime(closeFake), spec({
  654. stdio: { stdin: 'pipe', stdout: { maxBytes: 4 }, stderr: { maxBytes: 4 } },
  655. }), '/runtime/stdin-close-error')
  656. await flush()
  657. const closeError = once(closeHandle.stdin!, 'error')
  658. closeHandle.stdin!.end()
  659. await expect(closeError).resolves.toMatchObject([{ message: 'close rejected' }])
  660. closeFake.finish()
  661. await closeHandle.done
  662. })
  663. it('collects bounded tails, retains valid spills, and maps natural nonzero exits', async () => {
  664. const fake = new FakeSandbox()
  665. const handle = testHandle(runtime(fake), spec({
  666. stdio: {
  667. stdin: { data: 'batch' },
  668. stdout: { maxBytes: 4, spill: { maxBytes: 16 } },
  669. stderr: { maxBytes: 3 },
  670. },
  671. }), '/runtime/two')
  672. await flush()
  673. await fake.stdout('abcdef')
  674. await fake.stderr('12345')
  675. fake.finish(7)
  676. await expect(handle.done).resolves.toEqual({ exitCode: 7, signal: null })
  677. expect(fake.handle.sent).toEqual(['batch'])
  678. expect(fake.handle.closes).toBe(1)
  679. expect(handle.collected.stdout!.readFrom(0)).toEqual({
  680. text: 'cdef',
  681. nextOffset: 6,
  682. lossy: true,
  683. spillPath: '/runtime/two/stdout.log',
  684. })
  685. expect(handle.collected.stderr!.readFrom(0)).toEqual({ text: '345', nextOffset: 5, lossy: true })
  686. expect(fake.removed).not.toContain('/runtime/two/stdout.log')
  687. })
  688. it('removes a spill once the complete stream exceeds its cap', async () => {
  689. const fake = new FakeSandbox()
  690. const handle = testHandle(runtime(fake), spec({
  691. stdio: { stdin: 'ignore', stdout: { maxBytes: 2, spill: { maxBytes: 3 } }, stderr: 'inherit' },
  692. }), '/runtime/oversize')
  693. await flush()
  694. await fake.stdout('abcd')
  695. await fake.stderr('')
  696. fake.finish()
  697. await handle.done
  698. expect(handle.collected.stdout!.readFrom(0)).toEqual({ text: 'cd', nextOffset: 4, lossy: true })
  699. expect(fake.removed).toContain('/runtime/oversize/stdout.log')
  700. const command = fake.commandsSeen.find(value => value.includes('dsh_e2b_tee='))!
  701. expect(command).toContain('"$dsh_e2b_head" -c 3')
  702. expect(command).toContain('/runtime/oversize/stdout.log')
  703. expect(command).toContain('"$dsh_e2b_tee" --output-error=warn-nopipe')
  704. expect(command).not.toContain('tee -a')
  705. })
  706. it('contains remote spill-removal failures and routes empty inherited output', async () => {
  707. const fake = new FakeSandbox()
  708. fake.nextRemoveError = new Error('already removed')
  709. const handle = testHandle(runtime(fake), spec({
  710. stdio: { stdin: 'ignore', stdout: 'inherit', stderr: { maxBytes: 4, spill: { maxBytes: 8 } } },
  711. }), '/runtime/remove-error')
  712. await flush()
  713. await fake.stdout('')
  714. await fake.stderr('')
  715. fake.finish()
  716. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  717. expect(fake.removed).toContain('/runtime/remove-error/stderr.log')
  718. })
  719. it('terminates a process group with TERM and reports the signal outcome', async () => {
  720. const fake = new FakeSandbox()
  721. const handle = testHandle(runtime(fake), spec(), '/runtime/term')
  722. await flush()
  723. handle.terminate()
  724. handle.terminate()
  725. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGTERM' })
  726. await expect(handle.waitForExit()).resolves.toBe(true)
  727. expect(fake.commandsSeen).toContain('kill -TERM -- -4242')
  728. expect(fake.commandsSeen).not.toContain('kill -KILL -- -4242')
  729. const signals = fake.commandsSeen.filter(command => command.startsWith('kill -')).length
  730. fake.alive = true
  731. handle.terminate()
  732. await flush()
  733. expect(fake.alive).toBe(true)
  734. expect(fake.commandsSeen.filter(command => command.startsWith('kill -'))).toHaveLength(signals)
  735. })
  736. it('makes termination a permanent no-op after natural quiescence is observed', async () => {
  737. const fake = new FakeSandbox()
  738. const handle = testHandle(runtime(fake), spec(), '/runtime/natural-quiescence')
  739. await flush()
  740. fake.finish()
  741. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  742. await expect(handle.waitForExit()).resolves.toBe(true)
  743. const signals = fake.commandsSeen.filter(command => command.startsWith('kill -')).length
  744. fake.alive = true
  745. handle.terminate()
  746. await flush()
  747. expect(fake.alive).toBe(true)
  748. expect(fake.commandsSeen.filter(command => command.startsWith('kill -'))).toHaveLength(signals)
  749. })
  750. it('treats a zombie-only process group as quiescent', async () => {
  751. const fake = new FakeSandbox()
  752. fake.zombieOnly = true
  753. const handle = testHandle(runtime(fake), spec(), '/runtime/zombie-quiescence')
  754. await flush()
  755. await expect(handle.waitForExit()).resolves.toBe(true)
  756. expect(fake.commandsSeen).toContain(
  757. 'set -o pipefail; ps -eo pgid=,stat= | awk \'$1 == 4242 && $2 !~ /^[ZXx]/ { live=1 } END { if (live) print "live" }\'',
  758. )
  759. fake.finish()
  760. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  761. })
  762. it('keeps proven quiescence after a concurrent termination transport fails', async () => {
  763. const fake = new FakeSandbox()
  764. fake.signalErrors.push(new Error('TERM transport failed'), new Error('KILL transport failed'))
  765. fake.handle.killError = new Error('SDK kill failed')
  766. fake.deferSignals()
  767. const handle = testHandle(runtime(fake), spec(), '/runtime/quiescent-race')
  768. await flush()
  769. handle.terminate()
  770. await vi.waitFor(() => { expect(fake.commandsSeen).toContain('kill -TERM -- -4242') })
  771. fake.alive = false
  772. await expect(handle.waitForExit()).resolves.toBe(true)
  773. fake.probeError = new Error('post-quiescence probe failed')
  774. fake.releaseSignals()
  775. await vi.waitFor(() => { expect(fake.handle.kills).toBe(1) })
  776. fake.finish()
  777. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  778. const signals = fake.commandsSeen.filter(command => command.startsWith('kill -')).length
  779. handle.terminate()
  780. await expect(handle.waitForExit()).resolves.toBe(true)
  781. expect(fake.commandsSeen.filter(command => command.startsWith('kill -'))).toHaveLength(signals)
  782. })
  783. it('escalates a TERM-trapping process group to KILL and uses the SDK kill as fallback', async () => {
  784. const fake = new FakeSandbox()
  785. fake.trapsTerm = true
  786. fake.handle.killError = new Error('already gone')
  787. const handle = testHandle(runtime(fake), spec({ graceMs: 1 }), '/runtime/kill')
  788. await flush()
  789. handle.terminate()
  790. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGKILL' })
  791. await expect(handle.waitForExit()).resolves.toBe(true)
  792. expect(fake.commandsSeen).toContain('kill -KILL -- -4242')
  793. expect(fake.handle.kills).toBe(1)
  794. })
  795. it('keeps force cleanup retryable until quiescence is proven', async () => {
  796. const fake = new FakeSandbox()
  797. fake.trapsTerm = true
  798. fake.delaysKill = true
  799. fake.delaysKillCompletion = true
  800. fake.sdkKillStops = false
  801. const handle = testHandle(runtime(fake), spec({ graceMs: 1 }), '/runtime/termination-fence')
  802. await flush()
  803. handle.terminate()
  804. await vi.waitFor(() => { expect(fake.handle.kills).toBe(1) })
  805. await expect(handle.waitForExit()).rejects.toThrow('remained live after force termination')
  806. expect(fake.alive).toBe(true)
  807. fake.delaysKill = false
  808. fake.delaysKillCompletion = false
  809. fake.sdkKillStops = true
  810. handle.terminate()
  811. await expect(handle.waitForExit()).resolves.toBe(true)
  812. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGKILL' })
  813. })
  814. it('honors termination requested before asynchronous startup finishes', async () => {
  815. const fake = new FakeSandbox()
  816. fake.deferStart()
  817. const handle = testHandle(runtime(fake), spec(), '/runtime/deferred-kill')
  818. handle.terminate()
  819. fake.releaseStart()
  820. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGTERM' })
  821. })
  822. it('aborts a stalled preparation request before reporting startup quiescence', async () => {
  823. const fake = new FakeSandbox()
  824. let preparationSignal: AbortSignal | undefined
  825. fake.environmentRequest = async (signal) => {
  826. preparationSignal = signal
  827. await new Promise<never>((_resolve, reject) => {
  828. const rejectAbort = (): void => {
  829. const reason: unknown = signal?.reason
  830. reject(reason instanceof Error ? reason : new Error(String(reason)))
  831. }
  832. if (signal?.aborted === true) {
  833. rejectAbort()
  834. return
  835. }
  836. signal?.addEventListener('abort', rejectAbort, { once: true })
  837. })
  838. }
  839. const handle = testHandle(runtime(fake), spec(), '/runtime/stalled-preparation')
  840. await vi.waitFor(() => { expect(preparationSignal).toBeDefined() })
  841. handle.terminate()
  842. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGTERM' })
  843. await expect(handle.waitForExit()).resolves.toBe(true)
  844. expect(preparationSignal?.aborted).toBe(true)
  845. expect(fake.startOptions).toBeUndefined()
  846. })
  847. it('kills through the provisional SDK handle before process-group publication', async () => {
  848. const fake = new FakeSandbox()
  849. fake.deferProcessGroupRead()
  850. fake.signalErrors.push(commandError(1), commandError(1))
  851. const handle = testHandle(runtime(fake), spec(), '/runtime/pre-publication-kill')
  852. await vi.waitFor(() => { expect(fake.startOptions).toBeDefined() })
  853. handle.terminate()
  854. await vi.waitFor(() => { expect(fake.handle.kills).toBe(1) })
  855. expect(fake.alive).toBe(false)
  856. await expect(handle.waitForExit()).resolves.toBe(true)
  857. fake.releaseProcessGroupRead()
  858. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGKILL' })
  859. })
  860. it('does not treat an unsuccessful SDK fallback as provisional group quiescence', async () => {
  861. const fake = new FakeSandbox()
  862. fake.deferProcessGroupRead()
  863. fake.trapsTerm = true
  864. fake.delaysKill = true
  865. fake.delaysKillCompletion = true
  866. fake.sdkKillStops = false
  867. fake.handle.killResult = false
  868. const handle = testHandle(runtime(fake), spec({ graceMs: 1 }), '/runtime/provisional-sdk-false')
  869. await vi.waitFor(() => { expect(fake.startOptions).toBeDefined() })
  870. handle.terminate()
  871. await vi.waitFor(() => { expect(fake.handle.kills).toBe(1) })
  872. await expect(handle.waitForExit()).rejects.toThrow('remained live after force termination')
  873. fake.alive = false
  874. handle.terminate()
  875. await expect(handle.waitForExit()).resolves.toBe(true)
  876. fake.releaseProcessGroupRead()
  877. fake.finish()
  878. await handle.done
  879. })
  880. it('bounds a quiescence observer while provisional termination is awaiting the controller', async () => {
  881. const fake = new FakeSandbox()
  882. fake.deferProcessGroupRead()
  883. const reconnect = Promise.withResolvers<Sandbox>()
  884. let calls = 0
  885. const delayedRuntime = runtime(fake, async () => {
  886. calls += 1
  887. return calls === 1 ? fake.sandbox : await reconnect.promise
  888. })
  889. const handle = testHandle(delayedRuntime, spec(), '/runtime/pre-publication-observer')
  890. await vi.waitFor(() => { expect(fake.startOptions).toBeDefined() })
  891. handle.terminate()
  892. const controller = new AbortController()
  893. const waiting = handle.waitForExit(controller.signal)
  894. await flush()
  895. controller.abort()
  896. await expect(waiting).resolves.toBe(false)
  897. reconnect.resolve(fake.sandbox)
  898. await expect(handle.waitForExit()).resolves.toBe(true)
  899. fake.releaseProcessGroupRead()
  900. await handle.done
  901. })
  902. it('proves a provisional group exit when the SDK kill fallback fails', async () => {
  903. const fake = new FakeSandbox()
  904. fake.deferProcessGroupRead()
  905. fake.trapsTerm = true
  906. fake.handle.killError = new Error('SDK kill unavailable')
  907. const handle = testHandle(runtime(fake), spec({ graceMs: 1 }), '/runtime/pre-publication-group-kill')
  908. await vi.waitFor(() => { expect(fake.startOptions).toBeDefined() })
  909. handle.terminate()
  910. await expect(handle.waitForExit()).resolves.toBe(true)
  911. fake.releaseProcessGroupRead()
  912. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGKILL' })
  913. })
  914. it('reports failed provisional group and SDK force transports', async () => {
  915. const fake = new FakeSandbox()
  916. fake.deferProcessGroupRead()
  917. fake.signalErrors.push(new Error('TERM transport failed'), new Error('KILL transport failed'))
  918. fake.handle.killError = new Error('SDK kill failed')
  919. const handle = testHandle(runtime(fake), spec({ graceMs: 1 }), '/runtime/pre-publication-failure')
  920. await vi.waitFor(() => { expect(fake.startOptions).toBeDefined() })
  921. handle.terminate()
  922. await expect(handle.waitForExit()).rejects.toThrow('remained live after force termination')
  923. fake.handle.killError = undefined
  924. handle.terminate()
  925. await expect(handle.waitForExit()).resolves.toBe(true)
  926. fake.releaseProcessGroupRead()
  927. await handle.done
  928. const absentGroup = new FakeSandbox()
  929. absentGroup.deferProcessGroupRead()
  930. absentGroup.signalErrors.push(commandError(1), commandError(1))
  931. absentGroup.handle.killError = new Error('SDK kill failed without a provisional group')
  932. const absentHandle = testHandle(
  933. runtime(absentGroup),
  934. spec({ graceMs: 1 }),
  935. '/runtime/pre-publication-absent-group',
  936. )
  937. await vi.waitFor(() => { expect(absentGroup.startOptions).toBeDefined() })
  938. absentHandle.terminate()
  939. await expect(absentHandle.waitForExit()).rejects.toThrow('remained live after force termination')
  940. absentGroup.handle.killError = undefined
  941. absentHandle.terminate()
  942. await expect(absentHandle.waitForExit()).resolves.toBe(true)
  943. absentGroup.releaseProcessGroupRead()
  944. await absentHandle.done
  945. const optimisticSdk = new FakeSandbox()
  946. optimisticSdk.deferProcessGroupRead()
  947. optimisticSdk.signalErrors.push(commandError(1), commandError(1))
  948. optimisticSdk.sdkKillStops = false
  949. const optimisticHandle = testHandle(
  950. runtime(optimisticSdk),
  951. spec({ graceMs: 1 }),
  952. '/runtime/pre-publication-optimistic-sdk',
  953. )
  954. await vi.waitFor(() => { expect(optimisticSdk.startOptions).toBeDefined() })
  955. optimisticHandle.terminate()
  956. await expect(optimisticHandle.waitForExit()).rejects.toThrow('remained live after force termination')
  957. optimisticHandle.terminate()
  958. await expect(optimisticHandle.waitForExit()).resolves.toBe(true)
  959. optimisticSdk.releaseProcessGroupRead()
  960. await optimisticHandle.done
  961. })
  962. it('honors an already-aborted signal when constructing the asynchronous handle directly', async () => {
  963. const fake = new FakeSandbox()
  964. const handle = testHandle(runtime(fake), spec({ signal: AbortSignal.abort('stop') }), '/runtime/pre-aborted')
  965. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGTERM' })
  966. })
  967. it('reacts to a signal that aborts after the remote command has started', async () => {
  968. const fake = new FakeSandbox()
  969. const controller = new AbortController()
  970. const handle = testHandle(runtime(fake), spec({ signal: controller.signal }), '/runtime/live-abort')
  971. await flush()
  972. controller.abort('stop')
  973. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGTERM' })
  974. })
  975. it('can terminate a surviving process group after the command leader settles', async () => {
  976. const fake = new FakeSandbox()
  977. const handle = testHandle(runtime(fake), spec(), '/runtime/surviving-group')
  978. await flush()
  979. await fake.completeOutput()
  980. fake.handle.succeed(0)
  981. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  982. expect(fake.alive).toBe(true)
  983. handle.terminate()
  984. await flush()
  985. const signaled = fake.commandsSeen.includes('kill -TERM -- -4242')
  986. if (!signaled) fake.finish()
  987. await expect(handle.waitForExit()).resolves.toBe(true)
  988. expect(signaled).toBe(true)
  989. })
  990. it('bounds waitForExit while startup or a live group is pending', async () => {
  991. const fake = new FakeSandbox()
  992. fake.deferStart()
  993. const handle = testHandle(runtime(fake), spec(), '/runtime/wait')
  994. const beforeStart = new AbortController()
  995. const pending = handle.waitForExit(beforeStart.signal)
  996. beforeStart.abort()
  997. await expect(pending).resolves.toBe(false)
  998. await expect(handle.waitForExit(AbortSignal.abort())).resolves.toBe(false)
  999. fake.releaseStart()
  1000. await flush()
  1001. const live = new AbortController()
  1002. const liveWait = handle.waitForExit(live.signal)
  1003. live.abort()
  1004. await expect(liveWait).resolves.toBe(false)
  1005. fake.finish()
  1006. await handle.done
  1007. const terminatingFake = new FakeSandbox()
  1008. terminatingFake.deferStart()
  1009. const terminating = testHandle(runtime(terminatingFake), spec(), '/runtime/wait-termination-start')
  1010. terminating.terminate()
  1011. const beforeHandle = new AbortController()
  1012. const handlePending = terminating.waitForExit(beforeHandle.signal)
  1013. beforeHandle.abort()
  1014. await expect(handlePending).resolves.toBe(false)
  1015. terminatingFake.releaseStart()
  1016. await terminating.done
  1017. })
  1018. it('bounds both sides of the liveness-poll abort race', async () => {
  1019. const fake = new FakeSandbox()
  1020. const handle = testHandle(runtime(fake), spec(), '/runtime/poll-abort')
  1021. await flush()
  1022. const beforeTick = new AbortController()
  1023. fake.afterProbe = () => { beforeTick.abort(); fake.afterProbe = undefined }
  1024. await expect(handle.waitForExit(beforeTick.signal)).resolves.toBe(false)
  1025. const duringTick = new AbortController()
  1026. fake.afterProbe = () => {
  1027. fake.afterProbe = undefined
  1028. setTimeout(() => { duringTick.abort() }, 0)
  1029. }
  1030. await expect(handle.waitForExit(duringTick.signal)).resolves.toBe(false)
  1031. const duringProbe = new AbortController()
  1032. fake.beforeProbe = () => { duringProbe.abort(); fake.beforeProbe = undefined }
  1033. await expect(handle.waitForExit(duringProbe.signal)).resolves.toBe(false)
  1034. let racedAbort = false
  1035. const raceSignal = {
  1036. get aborted() { return racedAbort },
  1037. addEventListener: () => { racedAbort = true },
  1038. removeEventListener: () => {},
  1039. } as unknown as AbortSignal
  1040. await expect(handle.waitForExit(raceSignal)).resolves.toBe(false)
  1041. fake.finish()
  1042. await handle.done
  1043. })
  1044. it('observes a live group across one successful bounded poll', async () => {
  1045. const fake = new FakeSandbox()
  1046. const handle = testHandle(runtime(fake), spec(), '/runtime/poll-success')
  1047. await flush()
  1048. setTimeout(() => { fake.finish() }, 1)
  1049. await expect(handle.waitForExit(new AbortController().signal)).resolves.toBe(true)
  1050. await handle.done
  1051. })
  1052. it('treats startup failure as no live tree and contains readiness rejection', async () => {
  1053. const fake = new FakeSandbox()
  1054. fake.backgroundError = new Error('start failed')
  1055. const handle = testHandle(runtime(fake), spec(), '/runtime/fail')
  1056. await expect(handle.done).rejects.toThrow('start failed')
  1057. expect(handle.pid).toBeUndefined()
  1058. expect(fake.removed).toContain('/runtime/fail/environment')
  1059. expect(fake.removed).toContain('/runtime/fail')
  1060. await expect(handle.waitForExit()).resolves.toBe(true)
  1061. handle.terminate()
  1062. const unavailableHandle = testHandle(
  1063. runtime(new FakeSandbox(), async () => { throw new Error('sandbox unavailable') }),
  1064. spec(),
  1065. '/runtime/unavailable-start',
  1066. )
  1067. await expect(unavailableHandle.done).rejects.toThrow('sandbox unavailable')
  1068. await expect(unavailableHandle.waitForExit()).resolves.toBe(true)
  1069. const envFailure = new FakeSandbox()
  1070. envFailure.envError = new Error('ambient lookup failed')
  1071. const envHandle = testHandle(runtime(envFailure), spec(), '/runtime/env-failure')
  1072. await expect(envHandle.done).rejects.toThrow('ambient lookup failed')
  1073. expect(envFailure.removed).toEqual([])
  1074. const expectEnvironmentFailure = async (name: string, wire: string, message: string): Promise<void> => {
  1075. const fake = new FakeSandbox()
  1076. fake.environmentWire = wire
  1077. const failed = testHandle(runtime(fake), spec(), `/runtime/${name}`)
  1078. await expect(failed.done).rejects.toThrow(message)
  1079. }
  1080. const encodedEnvironment = Buffer.from('PATH=/bin\0').toString('base64')
  1081. const encodedHome = Buffer.from('/home/user').toString('base64')
  1082. await expectEnvironmentFailure('malformed-frame', '%', 'invalid base64')
  1083. await expectEnvironmentFailure('malformed-base64', `${encodedHome}\n%`, 'invalid base64')
  1084. await expectEnvironmentFailure(
  1085. 'invalid-utf8-home',
  1086. `${Buffer.from([0xff]).toString('base64')}\n${encodedEnvironment}`,
  1087. 'not valid UTF-8',
  1088. )
  1089. await expectEnvironmentFailure(
  1090. 'invalid-utf8-environment',
  1091. `${encodedHome}\n${Buffer.from([0xff]).toString('base64')}`,
  1092. 'not valid UTF-8',
  1093. )
  1094. await expectEnvironmentFailure(
  1095. 'relative-home',
  1096. `${Buffer.from('home/user').toString('base64')}\n${encodedEnvironment}`,
  1097. 'remote login home is invalid',
  1098. )
  1099. await expectEnvironmentFailure(
  1100. 'nul-home',
  1101. `${Buffer.from('/home/user\0tail').toString('base64')}\n${encodedEnvironment}`,
  1102. 'remote login home is invalid',
  1103. )
  1104. const cleanupFailure = new FakeSandbox()
  1105. cleanupFailure.backgroundError = new Error('start failed before credential consumption')
  1106. cleanupFailure.nextRemoveError = new Error('credential cleanup failed')
  1107. const cleanupHandle = testHandle(runtime(cleanupFailure), spec(), '/runtime/cleanup-failure')
  1108. await expect(cleanupHandle.done).rejects.toThrow('command failed and private state cleanup failed')
  1109. const absentState = new FakeSandbox()
  1110. absentState.backgroundError = new Error('start failed after external cleanup')
  1111. absentState.nextRemoveError = new FileNotFoundError('already removed')
  1112. const absentHandle = testHandle(runtime(absentState), spec(), '/runtime/absent-state')
  1113. await expect(absentHandle.done).rejects.toThrow('start failed after external cleanup')
  1114. })
  1115. it('bounds a readiness rejection with a still-live caller signal', async () => {
  1116. const fake = new FakeSandbox()
  1117. fake.deferStart()
  1118. fake.backgroundError = new Error('start failed')
  1119. const handle = testHandle(runtime(fake), spec(), '/runtime/fail-with-signal')
  1120. const waiting = handle.waitForExit(new AbortController().signal)
  1121. fake.releaseStart()
  1122. await expect(handle.done).rejects.toThrow('start failed')
  1123. await expect(waiting).resolves.toBe(true)
  1124. })
  1125. it('propagates an unavailable sandbox unless the caller aborts the wait', async () => {
  1126. const fake = new FakeSandbox()
  1127. let calls = 0
  1128. const unavailable = runtime(fake, async () => {
  1129. calls += 1
  1130. if (calls === 1) return fake.sandbox
  1131. throw new Error('connection unavailable')
  1132. })
  1133. const handle = testHandle(unavailable, spec(), '/runtime/unavailable')
  1134. await flush()
  1135. await expect(handle.waitForExit()).rejects.toThrow('connection unavailable')
  1136. fake.finish()
  1137. await handle.done
  1138. })
  1139. it('returns false when the caller aborts while reconnecting for liveness', async () => {
  1140. const fake = new FakeSandbox()
  1141. const reconnect = Promise.withResolvers<Sandbox>()
  1142. let calls = 0
  1143. const unavailable = runtime(fake, async () => {
  1144. calls += 1
  1145. return calls === 1 ? fake.sandbox : await reconnect.promise
  1146. })
  1147. const handle = testHandle(unavailable, spec(), '/runtime/reconnect-abort')
  1148. await flush()
  1149. const controller = new AbortController()
  1150. const waiting = handle.waitForExit(controller.signal)
  1151. await flush()
  1152. controller.abort()
  1153. reconnect.reject(new Error('connection unavailable'))
  1154. await expect(waiting).resolves.toBe(false)
  1155. fake.finish()
  1156. await handle.done
  1157. })
  1158. it('returns false when a liveness request itself is aborted and surfaces other probe failures', async () => {
  1159. const fake = new FakeSandbox()
  1160. const handle = testHandle(runtime(fake), spec(), '/runtime/probe')
  1161. await flush()
  1162. const controller = new AbortController()
  1163. controller.abort()
  1164. await expect(handle.waitForExit(controller.signal)).resolves.toBe(false)
  1165. fake.probeError = new Error('probe failed')
  1166. await expect(handle.waitForExit()).rejects.toThrow('probe failed')
  1167. fake.finish()
  1168. await handle.done
  1169. })
  1170. it('treats a timeout-killed sandbox as quiescent during liveness probing', async () => {
  1171. const fake = new FakeSandbox()
  1172. const handle = testHandle(runtime(fake), spec(), '/runtime/expired-sandbox')
  1173. await flush()
  1174. fake.finish()
  1175. await handle.done
  1176. fake.probeError = new SandboxNotFoundError('sandbox expired')
  1177. await expect(handle.waitForExit()).resolves.toBe(true)
  1178. })
  1179. it('treats a missing sandbox handle as quiescent during liveness acquisition', async () => {
  1180. const fake = new FakeSandbox()
  1181. let calls = 0
  1182. const handle = testHandle(runtime(fake, async () => {
  1183. calls += 1
  1184. if (calls === 1) return fake.sandbox
  1185. throw new SandboxNotFoundError('sandbox expired')
  1186. }), spec(), '/runtime/expired-acquisition')
  1187. await flush()
  1188. await expect(handle.waitForExit()).resolves.toBe(true)
  1189. await fake.completeOutput()
  1190. fake.alive = false
  1191. fake.handle.succeed(0)
  1192. await handle.done
  1193. })
  1194. it('treats sandbox loss during termination as quiescent', async () => {
  1195. const fake = new FakeSandbox()
  1196. let calls = 0
  1197. const handle = testHandle(runtime(fake, async () => {
  1198. calls += 1
  1199. if (calls === 1) return fake.sandbox
  1200. throw new SandboxNotFoundError('sandbox expired')
  1201. }), spec(), '/runtime/expired-termination')
  1202. await flush()
  1203. await fake.completeOutput()
  1204. handle.terminate()
  1205. await expect(handle.waitForExit()).resolves.toBe(true)
  1206. fake.alive = false
  1207. fake.handle.succeed(0)
  1208. await handle.done
  1209. })
  1210. it('makes batch stdin close failures best-effort', async () => {
  1211. const fake = new FakeSandbox()
  1212. vi.spyOn(fake.handle, 'sendStdin').mockRejectedValueOnce(new Error('closed'))
  1213. const handle = testHandle(runtime(fake), spec({
  1214. stdio: { stdin: { data: 'ignored' }, stdout: { maxBytes: 4 }, stderr: { maxBytes: 4 } },
  1215. }), '/runtime/stdin-closed')
  1216. await flush()
  1217. fake.finish()
  1218. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  1219. })
  1220. it('rejects malformed SDK process ids and non-command settlement failures', async () => {
  1221. const invalidPid = new FakeSandbox()
  1222. invalidPid.handle.pid = 0
  1223. const invalid = testHandle(runtime(invalidPid), spec(), '/runtime/invalid-pid')
  1224. await expect(invalid.done).rejects.toThrow(/invalid command pid 0/)
  1225. expect(invalidPid.handle.kills).toBe(1)
  1226. expect(invalidPid.removed).toContain('/runtime/invalid-pid/environment')
  1227. await expect(invalid.waitForExit()).resolves.toBe(true)
  1228. const failedRollback = new FakeSandbox()
  1229. failedRollback.handle.pid = 0
  1230. failedRollback.handle.killError = new Error('invalid handle kill failed')
  1231. const retained = testHandle(runtime(failedRollback), spec(), '/runtime/invalid-pid-retained')
  1232. await expect(retained.done).rejects.toThrow('invalid command pid rollback did not reach quiescence')
  1233. await expect(retained.waitForExit()).rejects.toThrow('invalid handle kill failed')
  1234. failedRollback.handle.killError = undefined
  1235. retained.terminate()
  1236. await expect(retained.waitForExit()).resolves.toBe(true)
  1237. const crashedFake = new FakeSandbox()
  1238. const crashed = testHandle(runtime(crashedFake), spec(), '/runtime/crashed')
  1239. await flush()
  1240. crashedFake.alive = false
  1241. crashedFake.handle.crash(new Error('command transport failed'))
  1242. await expect(crashed.done).rejects.toThrow('command transport failed')
  1243. })
  1244. it('rejects invalid or absent process-group publication', async () => {
  1245. const invalidGroup = new FakeSandbox()
  1246. invalidGroup.processGroupId = 'not-a-pid\n'
  1247. invalidGroup.delaysKill = true
  1248. invalidGroup.sdkKillStops = false
  1249. invalidGroup.afterProbe = () => { invalidGroup.alive = false }
  1250. const invalid = testHandle(runtime(invalidGroup), spec(), '/runtime/invalid-group')
  1251. await expect(invalid.done).rejects.toThrow(/invalid process-group id/)
  1252. expect(invalidGroup.handle.kills).toBe(1)
  1253. expect(invalidGroup.commandsSeen).toContain('kill -KILL -- -4242')
  1254. await expect(invalid.waitForExit()).resolves.toBe(true)
  1255. // A rewritten pid file must not aim the kill at every process (`-- -1`).
  1256. const unsafeGroup = new FakeSandbox()
  1257. unsafeGroup.processGroupId = '1\n'
  1258. unsafeGroup.delaysKill = true
  1259. unsafeGroup.sdkKillStops = false
  1260. unsafeGroup.afterProbe = () => { unsafeGroup.alive = false }
  1261. const unsafe = testHandle(runtime(unsafeGroup), spec(), '/runtime/unsafe-group')
  1262. await expect(unsafe.done).rejects.toThrow(/unsafe published process-group id 1/)
  1263. expect(unsafeGroup.commandsSeen).not.toContain('kill -KILL -- -1')
  1264. await expect(unsafe.waitForExit()).resolves.toBe(true)
  1265. const absentGroup = new FakeSandbox()
  1266. absentGroup.processGroupId = ''
  1267. const absent = testHandle(runtime(absentGroup), spec(), '/runtime/absent-group')
  1268. await flush()
  1269. absentGroup.finish()
  1270. await expect(absent.done).rejects.toThrow(/exited before publishing/)
  1271. expect(absentGroup.handle.kills).toBe(1)
  1272. expect(absentGroup.commandsSeen).toContain('kill -KILL -- -4242')
  1273. await expect(absent.waitForExit()).resolves.toBe(true)
  1274. })
  1275. it('preserves publication failure and reports cleanup that cannot be verified', async () => {
  1276. const fake = new FakeSandbox()
  1277. fake.processGroupId = 'not-a-pid\n'
  1278. fake.signalError = new Error('rollback signal failed')
  1279. fake.handle.killError = new Error('SDK kill failed')
  1280. const handle = testHandle(runtime(fake), spec(), '/runtime/failed-rollback')
  1281. let failure: unknown
  1282. try {
  1283. await handle.done
  1284. } catch (error: unknown) {
  1285. failure = error
  1286. }
  1287. expect(failure).toBeInstanceOf(AggregateError)
  1288. if (!(failure instanceof AggregateError)) throw new Error('expected AggregateError')
  1289. expect(failure.message).toBe('subprocess-e2b: process-group publication failed and rollback did not reach quiescence')
  1290. const failures = Array.from(failure.errors as Iterable<unknown>)
  1291. expect(failures).toHaveLength(2)
  1292. expect(failures[0]).toBeInstanceOf(Error)
  1293. expect(failures[1]).toBeInstanceOf(Error)
  1294. if (!(failures[0] instanceof Error) || !(failures[1] instanceof Error)) throw new Error('expected nested errors')
  1295. expect(failures[0].message).toContain('invalid process-group id')
  1296. expect(failures[1].message).toContain('remained live after force termination')
  1297. expect(fake.handle.kills).toBe(1)
  1298. const bounded = new AbortController()
  1299. const waiting = handle.waitForExit(bounded.signal)
  1300. bounded.abort()
  1301. await expect(waiting).resolves.toBe(false)
  1302. handle.terminate()
  1303. await expect(handle.waitForExit()).resolves.toBe(true)
  1304. expect(fake.commandsSeen).toContain('kill -TERM -- -4242')
  1305. const naturallyGone = new FakeSandbox()
  1306. naturallyGone.processGroupId = 'not-a-pid\n'
  1307. naturallyGone.signalError = new Error('rollback signal failed')
  1308. naturallyGone.handle.killError = new Error('SDK kill failed')
  1309. const observed = testHandle(runtime(naturallyGone), spec(), '/runtime/failed-rollback-observed')
  1310. await expect(observed.done).rejects.toThrow('process-group publication failed')
  1311. naturallyGone.alive = false
  1312. await expect(observed.waitForExit()).resolves.toBe(true)
  1313. })
  1314. it('keeps the public pid unavailable after delayed private process-group publication', async () => {
  1315. const fake = new FakeSandbox()
  1316. fake.processGroupReads.push('', '4242\n')
  1317. const handle = testHandle(runtime(fake), spec(), '/runtime/delayed-group')
  1318. await vi.waitFor(() => { expect(fake.processGroupReads).toHaveLength(0) })
  1319. expect(handle.pid).toBeUndefined()
  1320. fake.finish()
  1321. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  1322. })
  1323. it('handles output backpressure and contains a stderr sink failure', async () => {
  1324. const fake = new FakeSandbox()
  1325. const handle = testHandle(runtime(fake), spec({
  1326. stdio: { stdin: 'ignore', stdout: 'pipe', stderr: 'pipe' },
  1327. }), '/runtime/backpressure')
  1328. await flush()
  1329. handle.stdout!.on('error', () => {})
  1330. const stdoutWrite = vi.spyOn(handle.stdout!, 'write').mockReturnValueOnce(false)
  1331. const stdoutPending = fake.stdout('blocked')
  1332. queueMicrotask(() => { handle.stdout!.emit('drain') })
  1333. await stdoutPending
  1334. stdoutWrite.mockRestore()
  1335. handle.stderr!.on('error', () => {})
  1336. const stderrWrite = vi.spyOn(handle.stderr!, 'write').mockReturnValueOnce(false)
  1337. const stderrPending = fake.stderr('broken')
  1338. queueMicrotask(() => { handle.stderr!.emit('error', new Error('sink failed')) })
  1339. await stderrPending
  1340. stderrWrite.mockRestore()
  1341. fake.finish()
  1342. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  1343. })
  1344. it('settles output backpressure when the consumer closes the pipe', async () => {
  1345. const fake = new FakeSandbox()
  1346. const handle = testHandle(runtime(fake), spec({
  1347. stdio: { stdin: 'ignore', stdout: 'pipe', stderr: { maxBytes: 4 } },
  1348. }), '/runtime/backpressure-close')
  1349. await flush()
  1350. const stdoutWrite = vi.spyOn(handle.stdout!, 'write').mockReturnValueOnce(false)
  1351. const pending = fake.stdout('discarded')
  1352. queueMicrotask(() => { handle.stdout!.destroy() })
  1353. await pending
  1354. stdoutWrite.mockRestore()
  1355. fake.finish()
  1356. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  1357. })
  1358. it('breaks output backpressure when termination owns the command', async () => {
  1359. const fake = new FakeSandbox()
  1360. const handle = testHandle(runtime(fake), spec({
  1361. stdio: { stdin: 'ignore', stdout: 'pipe', stderr: { maxBytes: 4 } },
  1362. }), '/runtime/backpressure-termination')
  1363. await flush()
  1364. const stdoutWrite = vi.spyOn(handle.stdout!, 'write').mockReturnValueOnce(false)
  1365. let released = false
  1366. const pending = fake.stdout('blocked').then(() => { released = true })
  1367. await Promise.resolve()
  1368. handle.terminate()
  1369. await flush()
  1370. const releasedByTermination = released
  1371. if (!released) handle.stdout!.emit('drain')
  1372. await pending
  1373. stdoutWrite.mockRestore()
  1374. await handle.done
  1375. expect(releasedByTermination).toBe(true)
  1376. })
  1377. it('settles backpressure when a synchronous pipe write starts termination', async () => {
  1378. const fake = new FakeSandbox()
  1379. const handle = testHandle(runtime(fake), spec({
  1380. stdio: { stdin: 'ignore', stdout: 'pipe', stderr: { maxBytes: 4 } },
  1381. }), '/runtime/backpressure-synchronous-termination')
  1382. await flush()
  1383. const stdoutWrite = vi.spyOn(handle.stdout!, 'write').mockImplementationOnce(() => {
  1384. handle.terminate()
  1385. return false
  1386. })
  1387. await expect(fake.stdout('blocked')).resolves.toBeUndefined()
  1388. stdoutWrite.mockRestore()
  1389. await handle.done
  1390. })
  1391. it('contains a pipe callback failure instead of rejecting command settlement', async () => {
  1392. const fake = new FakeSandbox()
  1393. const handle = testHandle(runtime(fake), spec({
  1394. stdio: { stdin: 'ignore', stdout: 'pipe', stderr: { maxBytes: 4 } },
  1395. }), '/runtime/pipe-error')
  1396. await flush()
  1397. const emitted = once(handle.stdout!, 'error')
  1398. handle.stdout!.destroy(new Error('consumer failed'))
  1399. await emitted
  1400. await fake.stdout('late output')
  1401. fake.finish()
  1402. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  1403. })
  1404. it('contains an already-gone group signal and escalates after a TERM transport failure', async () => {
  1405. const gone = new FakeSandbox()
  1406. gone.trapsTerm = true
  1407. gone.signalError = commandError(1)
  1408. const goneHandle = testHandle(runtime(gone), spec({ graceMs: 1 }), '/runtime/gone-signal')
  1409. await flush()
  1410. goneHandle.terminate()
  1411. await expect(goneHandle.done).resolves.toEqual({ exitCode: null, signal: 'SIGKILL' })
  1412. const failed = new FakeSandbox()
  1413. failed.signalError = new Error('signal transport failed')
  1414. const failedHandle = testHandle(runtime(failed), spec(), '/runtime/failed-signal')
  1415. await flush()
  1416. failedHandle.terminate()
  1417. await expect(failedHandle.done).resolves.toEqual({ exitCode: null, signal: 'SIGKILL' })
  1418. expect(failed.commandsSeen).toContain('kill -KILL -- -4242')
  1419. })
  1420. it('allows termination retry after both force transports fail', async () => {
  1421. const fake = new FakeSandbox()
  1422. fake.signalErrors.push(new Error('TERM transport failed'), new Error('KILL transport failed'))
  1423. fake.handle.killError = new Error('SDK kill failed')
  1424. const handle = testHandle(runtime(fake), spec({ graceMs: 1 }), '/runtime/retry-signal')
  1425. await flush()
  1426. handle.terminate()
  1427. await expect(handle.waitForExit()).rejects.toThrow('remained live after force termination')
  1428. fake.handle.killError = undefined
  1429. handle.terminate()
  1430. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGTERM' })
  1431. expect(fake.commandsSeen.filter(command => command.startsWith('kill -TERM '))).toHaveLength(2)
  1432. const missingGroup = new FakeSandbox()
  1433. missingGroup.trapsTerm = true
  1434. missingGroup.signalErrors.push(undefined, commandError(1))
  1435. missingGroup.handle.killError = new Error('SDK kill failed after group exit race')
  1436. const raced = testHandle(runtime(missingGroup), spec({ graceMs: 1 }), '/runtime/group-exit-race')
  1437. await flush()
  1438. raced.terminate()
  1439. await expect(raced.waitForExit()).rejects.toThrow('remained live after force termination')
  1440. missingGroup.handle.killError = undefined
  1441. raced.terminate()
  1442. await expect(raced.waitForExit()).resolves.toBe(true)
  1443. })
  1444. it('rejects an optimistic SDK kill while descendants survive a failed group KILL', async () => {
  1445. const fake = new FakeSandbox()
  1446. fake.trapsTerm = true
  1447. fake.sdkKillStops = false
  1448. fake.signalErrors.push(undefined, new Error('KILL transport failed'))
  1449. const handle = testHandle(runtime(fake), spec({ graceMs: 1 }), '/runtime/optimistic-sdk-kill')
  1450. await flush()
  1451. handle.terminate()
  1452. await expect(handle.waitForExit()).rejects.toThrow('remained live after force termination')
  1453. expect(fake.alive).toBe(true)
  1454. fake.sdkKillStops = true
  1455. handle.terminate()
  1456. await expect(handle.waitForExit()).resolves.toBe(true)
  1457. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGKILL' })
  1458. })
  1459. })
  1460. describe('E2BSubprocessRuntime', () => {
  1461. async function service(
  1462. fake = new FakeSandbox(),
  1463. providedRuntime: E2BRuntime = runtime(fake),
  1464. ): Promise<{ ctx: Context; fiber: Awaited<ReturnType<Context['plugin']>> }> {
  1465. const ctx = new Context()
  1466. ctx.provide('e2b', providedRuntime)
  1467. const fiber = await ctx.plugin(E2BSubprocessRuntime)
  1468. return { ctx, fiber }
  1469. }
  1470. it('registers handles and disposal terminates and joins live remote groups regardless of sandbox policy', async () => {
  1471. const fake = new FakeSandbox()
  1472. fake.trapsTerm = true
  1473. const { ctx, fiber } = await service(fake)
  1474. const handle = ctx.subprocess.spawn(spec({ graceMs: 1 }))
  1475. await flush()
  1476. await fiber.dispose()
  1477. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGKILL' })
  1478. expect(fake.alive).toBe(false)
  1479. })
  1480. it('awaits SDK settlement after the remote process group becomes quiescent', async () => {
  1481. const fake = new FakeSandbox()
  1482. fake.trapsTerm = true
  1483. const { ctx, fiber } = await service(fake)
  1484. const handle = ctx.subprocess.spawn(spec())
  1485. await flush()
  1486. fake.alive = false
  1487. let disposed = false
  1488. const disposing = fiber.dispose().then(() => { disposed = true })
  1489. await flush()
  1490. expect(disposed).toBe(false)
  1491. fake.finish()
  1492. await disposing
  1493. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  1494. })
  1495. it('reports a failed termination transaction from disposal instead of waiting on done', async () => {
  1496. const fake = new FakeSandbox()
  1497. fake.signalErrors.push(new Error('TERM transport failed'), new Error('KILL transport failed'))
  1498. fake.handle.killError = new Error('SDK kill failed')
  1499. const { ctx, fiber } = await service(fake)
  1500. const handle = ctx.subprocess.spawn(spec({ graceMs: 1 }))
  1501. await flush()
  1502. await expect(fiber.dispose()).resolves.toBeUndefined()
  1503. await expect(handle.waitForExit()).rejects.toThrow('remained live after force termination')
  1504. fake.handle.killError = undefined
  1505. handle.terminate()
  1506. await expect(handle.waitForExit()).resolves.toBe(true)
  1507. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGTERM' })
  1508. })
  1509. it('aggregates sibling cleanup failures instead of reporting only the first', async () => {
  1510. const { ctx, fiber } = await service()
  1511. const disposalErrors: unknown[] = []
  1512. ctx.logger.error = ((error: unknown) => { disposalErrors.push(error) }) as typeof ctx.logger.error
  1513. const first = {
  1514. terminate: vi.fn(),
  1515. waitForExit: vi.fn(async () => { throw new Error('first cleanup failed') }),
  1516. done: Promise.resolve({ exitCode: 0, signal: null }),
  1517. } as unknown as E2BSubprocessHandle
  1518. const second = {
  1519. terminate: vi.fn(),
  1520. waitForExit: vi.fn(async () => { throw new Error('second cleanup failed') }),
  1521. done: Promise.resolve({ exitCode: 0, signal: null }),
  1522. } as unknown as E2BSubprocessHandle
  1523. const live = (ctx.subprocess as unknown as { live: Set<E2BSubprocessHandle> }).live
  1524. live.add(first)
  1525. live.add(second)
  1526. await fiber.dispose()
  1527. const failure = disposalErrors[0]
  1528. expect(failure).toBeInstanceOf(AggregateError)
  1529. if (!(failure instanceof AggregateError)) throw new Error('expected AggregateError')
  1530. expect(failure.errors.map(error => (error as Error).message).sort()).toEqual([
  1531. 'first cleanup failed',
  1532. 'second cleanup failed',
  1533. ])
  1534. })
  1535. it('waits for every owned cleanup before reporting a disposal failure', async () => {
  1536. const { ctx, fiber } = await service()
  1537. const failed = {
  1538. terminate: vi.fn(),
  1539. waitForExit: vi.fn(async () => { throw new Error('cleanup failed') }),
  1540. done: Promise.resolve({ exitCode: 0, signal: null }),
  1541. } as unknown as E2BSubprocessHandle
  1542. let finishCleanup!: () => void
  1543. const cleanup = new Promise<boolean>((resolve) => {
  1544. finishCleanup = () => { resolve(true) }
  1545. })
  1546. const draining = {
  1547. terminate: vi.fn(),
  1548. waitForExit: vi.fn(() => cleanup),
  1549. done: Promise.resolve({ exitCode: 0, signal: null }),
  1550. } as unknown as E2BSubprocessHandle
  1551. const live = (ctx.subprocess as unknown as { live: Set<E2BSubprocessHandle> }).live
  1552. live.add(failed)
  1553. live.add(draining)
  1554. let disposed = false
  1555. const disposing = fiber.dispose().then(() => { disposed = true })
  1556. await flush()
  1557. expect(disposed).toBe(false)
  1558. finishCleanup()
  1559. await disposing
  1560. expect(live).toEqual(new Set([failed]))
  1561. })
  1562. it('releases naturally settled handles before later service disposal', async () => {
  1563. const fake = new FakeSandbox()
  1564. const { ctx, fiber } = await service(fake)
  1565. const handle = ctx.subprocess.spawn(spec())
  1566. await flush()
  1567. fake.finish()
  1568. await handle.done
  1569. await flush()
  1570. const signalsBefore = fake.commandsSeen.filter(command => command.startsWith('kill -')).length
  1571. await fiber.dispose()
  1572. expect(fake.commandsSeen.filter(command => command.startsWith('kill -')).length).toBe(signalsBefore)
  1573. })
  1574. it('contains a release liveness failure and retries quiescence during disposal', async () => {
  1575. const fake = new FakeSandbox()
  1576. let calls = 0
  1577. const reconnecting = runtime(fake, async () => {
  1578. calls += 1
  1579. if (calls === 2) throw new Error('transient liveness failure')
  1580. return fake.sandbox
  1581. })
  1582. const { ctx, fiber } = await service(fake, reconnecting)
  1583. const handle = ctx.subprocess.spawn(spec())
  1584. await flush()
  1585. fake.finish()
  1586. await handle.done
  1587. await flush()
  1588. await fiber.dispose()
  1589. expect(calls).toBeGreaterThanOrEqual(3)
  1590. })
  1591. it('contains spawn rejection while disposal is joining the pending handle', async () => {
  1592. const fake = new FakeSandbox()
  1593. fake.deferStart()
  1594. fake.backgroundError = new Error('start failed during disposal')
  1595. const { ctx, fiber } = await service(fake)
  1596. const subprocess = ctx.subprocess
  1597. const handle = subprocess.spawn(spec())
  1598. await vi.waitFor(() => { expect(fake.startOptions).toBeDefined() })
  1599. const disposing = fiber.dispose()
  1600. await flush()
  1601. expect(() => subprocess.spawn(spec())).toThrow('service is disposing')
  1602. fake.releaseStart()
  1603. await expect(disposing).resolves.toBeUndefined()
  1604. await expect(handle.done).rejects.toThrow('start failed during disposal')
  1605. })
  1606. it('validates synchronous spawn preconditions', async () => {
  1607. const { ctx } = await service()
  1608. expect(() => ctx.subprocess.spawn(spec({ argv: [] }))).toThrow(/non-empty program/)
  1609. expect(() => ctx.subprocess.spawn(spec({ signal: AbortSignal.abort('stop') }))).toThrow(/aborted before spawn/)
  1610. })
  1611. it('registers the package-owned empty invariant installer', async () => {
  1612. const ctx = new Context()
  1613. await ctx.plugin(InvariantRegistry, { enabled: true })
  1614. const fiber = await ctx.plugin(E2BSubprocessInvariant).await()
  1615. await fiber.dispose()
  1616. })
  1617. })