subprocess.spec.ts 72 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810
  1. import { once } from 'node:events'
  2. import { Context } from '@deepseek-ai/cordis'
  3. import {
  4. CommandExitError,
  5. FileNotFoundError,
  6. SandboxNotFoundError,
  7. type CommandHandle,
  8. type CommandResult,
  9. type Sandbox,
  10. } from '@deepseek-ai/dsh-e2b'
  11. import type E2BRuntime from '@deepseek-ai/dsh-e2b'
  12. import type { SubprocessSpawnSpec } from '@deepseek-ai/dsh-subprocess'
  13. import E2BSubprocessRuntime from '@deepseek-ai/dsh-subprocess-e2b'
  14. import { E2BBase64Decoder, E2B_OUTPUT_COMPLETE_FRAME, E2BOutputReader } from '../src/output.ts'
  15. import { E2BSubprocessHandle } from '../src/process.ts'
  16. import { describe, expect, it, vi } from 'vitest'
  17. function commandError(exitCode: number): CommandExitError {
  18. return new CommandExitError({ exitCode, stdout: '', stderr: '', error: `exit ${exitCode}` })
  19. }
  20. interface StartOptions {
  21. background: true
  22. cwd: string
  23. stdin: boolean
  24. timeoutMs: number
  25. signal?: AbortSignal
  26. envs?: Record<string, string>
  27. onStdout?: (data: string) => void | Promise<void>
  28. onStderr?: (data: string) => void | Promise<void>
  29. }
  30. class FakeCommandHandle {
  31. pid = 4242
  32. readonly sent: Array<string | Uint8Array> = []
  33. closes = 0
  34. kills = 0
  35. disconnects = 0
  36. killError: unknown
  37. killResult = true
  38. disconnectError: unknown
  39. private readonly result = Promise.withResolvers<CommandResult>()
  40. private settled = false
  41. constructor(private readonly onKill: () => void = () => {}) {}
  42. wait(): Promise<CommandResult> {
  43. return this.result.promise
  44. }
  45. async sendStdin(data: string | Uint8Array): Promise<void> {
  46. this.sent.push(data)
  47. }
  48. async closeStdin(): Promise<void> {
  49. this.closes += 1
  50. }
  51. async kill(): Promise<boolean> {
  52. this.kills += 1
  53. if (this.killError !== undefined) throw this.killError
  54. this.onKill()
  55. return this.killResult
  56. }
  57. async disconnect(): Promise<void> {
  58. this.disconnects += 1
  59. if (this.disconnectError !== undefined) throw this.disconnectError
  60. }
  61. succeed(exitCode = 0): void {
  62. if (this.settled) return
  63. this.settled = true
  64. this.result.resolve({ exitCode, stdout: '', stderr: '' })
  65. }
  66. fail(exitCode: number): void {
  67. if (this.settled) return
  68. this.settled = true
  69. this.result.reject(commandError(exitCode))
  70. }
  71. crash(error: unknown): void {
  72. if (this.settled) return
  73. this.settled = true
  74. this.result.reject(error)
  75. }
  76. }
  77. class FakeSandbox {
  78. readonly handle: FakeCommandHandle
  79. readonly commandsSeen: string[] = []
  80. readonly writtenFiles: string[][] = []
  81. readonly writtenFileData = new Map<string, string>()
  82. readonly removed: string[] = []
  83. readonly directories: string[] = []
  84. startOptions: StartOptions | undefined
  85. backgroundError: unknown
  86. envError: unknown
  87. statusError: unknown
  88. nextRemoveError: unknown
  89. probeError: unknown
  90. signalError: unknown
  91. readonly signalErrors: unknown[] = []
  92. trapsTerm = false
  93. delaysKill = false
  94. delaysKillCompletion = false
  95. sdkKillStops = true
  96. alive = true
  97. zombieOnly = false
  98. ambient = 'PATH=/ambient/bin\0KEEP=safe\0UNICODE=你好\0NPM_TOKEN=secret\0DSH_STALE=old\0BROKEN\0=bad\0'
  99. environmentHome = '/home/user'
  100. environmentWire: string | undefined
  101. environmentRequest: ((signal: AbortSignal | undefined) => Promise<void>) | undefined
  102. processGroupId = '4242\n'
  103. exitStatus = ''
  104. statusReads = 0
  105. readonly processGroupReads: string[] = []
  106. afterStatusRead: (() => void) | undefined
  107. beforeProbe: (() => void) | undefined
  108. afterProbe: (() => void) | undefined
  109. private startGate: Promise<void> | undefined
  110. private openStart: (() => void) | undefined
  111. private processGroupReadGate: Promise<void> | undefined
  112. private openProcessGroupRead: (() => void) | undefined
  113. private signalGate: Promise<void> | undefined
  114. private openSignal: (() => void) | undefined
  115. constructor() {
  116. this.handle = new FakeCommandHandle(() => {
  117. if (this.sdkKillStops) {
  118. this.alive = false
  119. this.handle.fail(137)
  120. }
  121. })
  122. }
  123. deferStart(): void {
  124. const gate = Promise.withResolvers<undefined>()
  125. this.startGate = gate.promise
  126. this.openStart = () => { gate.resolve(undefined) }
  127. }
  128. releaseStart(): void {
  129. this.openStart?.()
  130. }
  131. deferProcessGroupRead(): void {
  132. const gate = Promise.withResolvers<undefined>()
  133. this.processGroupReadGate = gate.promise
  134. this.openProcessGroupRead = () => { gate.resolve(undefined) }
  135. }
  136. releaseProcessGroupRead(): void {
  137. this.openProcessGroupRead?.()
  138. }
  139. deferSignals(): void {
  140. const gate = Promise.withResolvers<undefined>()
  141. this.signalGate = gate.promise
  142. this.openSignal = () => { gate.resolve(undefined) }
  143. }
  144. releaseSignals(): void {
  145. this.openSignal?.()
  146. }
  147. finish(exitCode = 0): void {
  148. this.alive = false
  149. void this.completeOutput().then(
  150. () => {
  151. if (exitCode === 0) this.handle.succeed(0)
  152. else this.handle.fail(exitCode)
  153. },
  154. (error: unknown) => { this.handle.crash(error) },
  155. )
  156. }
  157. async completeOutput(): Promise<void> {
  158. await Promise.all([
  159. this.stdoutWire(`${E2B_OUTPUT_COMPLETE_FRAME}\n`),
  160. this.stderrWire(`${E2B_OUTPUT_COMPLETE_FRAME}\n`),
  161. ])
  162. }
  163. async stdout(data: string): Promise<void> {
  164. await this.stdoutWire(data.length === 0 ? '' : `${Buffer.from(data).toString('base64')}\n`)
  165. }
  166. async stderr(data: string): Promise<void> {
  167. await this.stderrWire(data.length === 0 ? '' : `${Buffer.from(data).toString('base64')}\n`)
  168. }
  169. async stdoutWire(data: string): Promise<void> {
  170. await this.startOptions?.onStdout?.(data)
  171. }
  172. async stderrWire(data: string): Promise<void> {
  173. await this.startOptions?.onStderr?.(data)
  174. }
  175. readonly sandbox = {
  176. sandboxId: 'fake',
  177. files: {
  178. makeDir: async (path: string): Promise<boolean> => {
  179. this.directories.push(path)
  180. return true
  181. },
  182. write: async (files: Array<{ path: string; data: string }>): Promise<object[]> => {
  183. this.writtenFiles.push(files.map(file => file.path))
  184. for (const file of files) this.writtenFileData.set(file.path, file.data)
  185. return files.map(() => ({}))
  186. },
  187. read: async (path: string): Promise<string> => {
  188. if (!path.endsWith('/exit-code')) {
  189. await this.processGroupReadGate
  190. return this.processGroupReads.shift() ?? this.processGroupId
  191. }
  192. if (this.statusError !== undefined) {
  193. const error = this.statusError
  194. this.statusError = undefined
  195. throw error
  196. }
  197. this.statusReads += 1
  198. this.afterStatusRead?.()
  199. return this.exitStatus
  200. },
  201. remove: async (path: string): Promise<void> => {
  202. this.removed.push(path)
  203. if (this.nextRemoveError !== undefined) {
  204. const error = this.nextRemoveError
  205. this.nextRemoveError = undefined
  206. throw error
  207. }
  208. },
  209. },
  210. commands: {
  211. run: async (command: string, options?: StartOptions | { signal?: AbortSignal }): Promise<CommandHandle | CommandResult> => {
  212. this.commandsSeen.push(command)
  213. if (command.includes('env -0 | base64')) {
  214. await this.environmentRequest?.(options?.signal)
  215. if (this.envError !== undefined) throw this.envError
  216. return {
  217. exitCode: 0,
  218. stdout: this.environmentWire ?? [this.environmentHome, this.ambient]
  219. .map(value => Buffer.from(value).toString('base64'))
  220. .join('\n'),
  221. stderr: '',
  222. }
  223. }
  224. if (command.startsWith('set -o pipefail; ps -eo pgid=,stat=')) {
  225. this.beforeProbe?.()
  226. if (options?.signal?.aborted === true) throw new DOMException('aborted', 'AbortError')
  227. if (this.probeError !== undefined) {
  228. const error = this.probeError
  229. this.probeError = undefined
  230. throw error
  231. }
  232. const stdout = this.alive && !this.zombieOnly ? 'live\n' : ''
  233. this.afterProbe?.()
  234. return { exitCode: 0, stdout, stderr: '' }
  235. }
  236. if (command.startsWith('kill -TERM ')) {
  237. await this.signalGate
  238. const error = this.signalErrors.shift() ?? this.signalError
  239. if (error !== undefined) {
  240. if (this.signalErrors.length === 0) this.signalError = undefined
  241. throw error
  242. }
  243. if (!this.trapsTerm) {
  244. this.alive = false
  245. this.handle.fail(143)
  246. }
  247. return { exitCode: 0, stdout: '', stderr: '' }
  248. }
  249. if (command.startsWith('kill -KILL ')) {
  250. await this.signalGate
  251. const error = this.signalErrors.shift() ?? this.signalError
  252. if (error !== undefined) {
  253. if (this.signalErrors.length === 0) this.signalError = undefined
  254. throw error
  255. }
  256. if (!this.delaysKill) this.alive = false
  257. if (!this.delaysKillCompletion) this.handle.fail(137)
  258. return { exitCode: 0, stdout: '', stderr: '' }
  259. }
  260. if ((options as StartOptions | undefined)?.background === true) {
  261. this.startOptions = options as StartOptions
  262. await this.startGate
  263. if (this.backgroundError !== undefined) throw this.backgroundError
  264. return this.handle as unknown as CommandHandle
  265. }
  266. return { exitCode: 0, stdout: '', stderr: '' }
  267. },
  268. },
  269. } as unknown as Sandbox
  270. }
  271. function spec(overrides: Partial<SubprocessSpawnSpec> = {}): SubprocessSpawnSpec {
  272. return {
  273. argv: ['bash', '-c', 'printf ok'],
  274. cwd: '/workspace',
  275. stdio: {
  276. stdin: 'ignore',
  277. stdout: { maxBytes: 4, spill: { maxBytes: 16 } },
  278. stderr: { maxBytes: 4 },
  279. },
  280. graceMs: 5,
  281. ...overrides,
  282. }
  283. }
  284. function runtime(fake: FakeSandbox, getSandbox: () => Promise<Sandbox> = async () => fake.sandbox): E2BRuntime {
  285. return {
  286. cwd: '/workspace',
  287. runtimeRoot: '/workspace/.dsh-e2b',
  288. getSandbox,
  289. } as unknown as E2BRuntime
  290. }
  291. async function flush(): Promise<void> {
  292. await new Promise(resolve => setTimeout(resolve, 0))
  293. }
  294. /** Construct the handle under test with the config default the service would pass. */
  295. function testHandle(
  296. runtime: ConstructorParameters<typeof E2BSubprocessHandle>[0],
  297. spec: ConstructorParameters<typeof E2BSubprocessHandle>[1],
  298. stateDir: string,
  299. pollMs = 20,
  300. ): E2BSubprocessHandle {
  301. return new E2BSubprocessHandle(runtime, spec, stateDir, pollMs)
  302. }
  303. describe('E2BOutputReader', () => {
  304. it('decodes base64 across arbitrary callback boundaries and rejects malformed framing', () => {
  305. const decoder = new E2BBase64Decoder()
  306. expect(decoder.push('')).toEqual(Buffer.alloc(0))
  307. expect(decoder.push('5')).toEqual(Buffer.alloc(0))
  308. expect(decoder.push('L2')).toEqual(Buffer.alloc(0))
  309. expect(decoder.push('g\n').toString()).toBe('你')
  310. expect(decoder.push('YQ==\nYg==\n').toString()).toBe('ab')
  311. expect(decoder.push(`${Buffer.from([0, 255]).toString('base64')}\n`)).toEqual(Buffer.from([0, 255]))
  312. expect(decoder.push(`${E2B_OUTPUT_COMPLETE_FRAME}\n`)).toEqual(Buffer.alloc(0))
  313. decoder.finish()
  314. expect(() => new E2BBase64Decoder().push('%\n')).toThrow('invalid base64')
  315. expect(() => new E2BBase64Decoder().push('AB==\n')).toThrow('invalid base64')
  316. expect(() => decoder.push(`${E2B_OUTPUT_COMPLETE_FRAME}\n`)).toThrow('duplicate output transport completion')
  317. expect(() => decoder.push('YQ==\n')).toThrow('continued after completion')
  318. const truncated = new E2BBase64Decoder()
  319. truncated.push('YQ')
  320. expect(() => { truncated.finish() }).toThrow('truncated base64')
  321. expect(() => { new E2BBase64Decoder().finish() }).toThrow('incomplete output transport')
  322. const interrupted = new E2BBase64Decoder()
  323. interrupted.push('YQ')
  324. expect(() => { interrupted.finish(false) }).not.toThrow()
  325. })
  326. it('keeps a byte-exact tail with independent whole-stream cursors', () => {
  327. const reader = new E2BOutputReader(4, 10, '/remote/spill')
  328. reader.push(Buffer.alloc(0))
  329. reader.push(Buffer.from('ab'))
  330. reader.push(Buffer.from('cdef'))
  331. expect(reader.size).toBe(6)
  332. expect(reader.readFrom(0)).toEqual({ text: 'cdef', nextOffset: 6, lossy: true, spillPath: '/remote/spill' })
  333. expect(reader.readFrom(2)).toEqual({ text: 'cdef', nextOffset: 6, lossy: false })
  334. expect(reader.readFrom(5)).toEqual({ text: 'f', nextOffset: 6, lossy: false })
  335. expect(reader.readFrom(99)).toEqual({ text: '', nextOffset: 6, lossy: false })
  336. reader.invalidateSpill()
  337. expect(reader.readFrom(0)).toEqual({ text: 'cdef', nextOffset: 6, lossy: true })
  338. })
  339. it('drops whole head chunks and withholds absent or over-cap spills', () => {
  340. const withoutSpill = new E2BOutputReader(2, undefined, '/unused')
  341. withoutSpill.push(Buffer.from('ab'))
  342. withoutSpill.push(Buffer.from('cd'))
  343. expect(withoutSpill.readFrom(0)).toEqual({ text: 'cd', nextOffset: 4, lossy: true })
  344. const overCap = new E2BOutputReader(2, 3, '/too-small')
  345. overCap.push(Buffer.from('abcd'))
  346. expect(overCap.readFrom(0)).toEqual({ text: 'cd', nextOffset: 4, lossy: true })
  347. })
  348. })
  349. describe('E2BSubprocessHandle', () => {
  350. it('starts asynchronously, keeps secrets out of the command, and supports deferred piped stdin/output', async () => {
  351. const fake = new FakeSandbox()
  352. fake.processGroupId = '4343\n'
  353. fake.deferStart()
  354. const handle = testHandle(runtime(fake), spec({
  355. argv: ['tool', 'argument with spaces'],
  356. stdio: { stdin: 'pipe', stdout: 'pipe', stderr: { maxBytes: 8, spill: { maxBytes: 32 } } },
  357. env: {
  358. PATH: '/bin',
  359. 'FOO-BAR': 'hyphen-value',
  360. '--split-string': 'literal-value',
  361. DEEPSEEK_API_KEY: 'explicit-secret',
  362. DSH_MODE: 'test',
  363. // The seam's tombstone: an explicit undefined removes the ambient entry.
  364. KEEP: undefined,
  365. },
  366. }), '/workspace/.dsh-e2b/processes/one')
  367. handle.stdin!.write('hello')
  368. handle.stdin!.end()
  369. fake.releaseStart()
  370. await flush()
  371. expect(fake.handle.sent.map(value => String(value))).toEqual(['hello'])
  372. expect(fake.handle.closes).toBe(1)
  373. const controlEnvs = fake.startOptions?.envs
  374. expect(controlEnvs?.HOME).toMatch(/^\/\.dsh-e2b-control-/)
  375. expect(controlEnvs).toEqual({
  376. TERM: 'dumb',
  377. NPM_TOKEN: '',
  378. DSH_STALE: '',
  379. HOME: controlEnvs?.HOME,
  380. })
  381. const command = fake.commandsSeen.find(value => value.includes('exec "$dsh_e2b_env_bin" -i'))!
  382. expect(command).toContain('"$dsh_e2b_setsid" --wait -- "$dsh_e2b_bash" -c')
  383. expect(command).not.toContain('DEEPSEEK_API_KEY')
  384. expect(command).not.toContain('DSH_MODE')
  385. expect(command).not.toContain('FOO-BAR')
  386. expect(command).not.toContain('explicit-secret')
  387. expect(command).not.toContain('hyphen-value')
  388. expect(command).not.toContain('${!dsh_e2b_name}')
  389. const environmentProbe = fake.commandsSeen.find(value => value.includes('env -0 | base64'))
  390. expect(environmentProbe).toContain('getent passwd "$(id -u)"')
  391. expect(environmentProbe).toContain('test -n "$dsh_e2b_home" -a -d "$dsh_e2b_home"')
  392. expect(environmentProbe).not.toContain('"$PWD"')
  393. expect(command).toContain('mapfile -d')
  394. expect(command).toContain('dsh_e2b_node="$(command -v node)"')
  395. expect(command).toContain('"$dsh_e2b_env_bin" -i "$dsh_e2b_node" -e')
  396. expect(command).toContain('"$dsh_e2b_env_bin" -i -- "${dsh_e2b_env[@]}" "$@"')
  397. expect(command).toContain('exec "$dsh_e2b_env_bin" -i -- "${dsh_e2b_env[@]}"')
  398. expect(command).toContain('>&2 2>/dev/null')
  399. expect(command).not.toContain('2>/dev/null >&2')
  400. expect(command).toContain('base64')
  401. expect(fake.writtenFiles[0]).toEqual([
  402. '/workspace/.dsh-e2b/processes/one/pid',
  403. '/workspace/.dsh-e2b/processes/one/exit-code',
  404. '/workspace/.dsh-e2b/processes/one/environment',
  405. '/workspace/.dsh-e2b/processes/one/stderr.log',
  406. ])
  407. expect(fake.writtenFileData.get('/workspace/.dsh-e2b/processes/one/environment')).toBe(
  408. 'PATH=/bin\0UNICODE=你好\0HOME=/home/user\0FOO-BAR=hyphen-value\0--split-string=literal-value\0DEEPSEEK_API_KEY=explicit-secret\0DSH_MODE=test\0',
  409. )
  410. let piped = ''
  411. handle.stdout!.on('data', (chunk) => { piped += String(chunk) })
  412. await fake.stdout('pipe-data')
  413. await fake.stderr('err')
  414. fake.finish()
  415. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  416. expect(piped).toBe('pipe-data')
  417. expect(handle.collected.stderr!.readFrom(0)).toMatchObject({ text: 'err', lossy: false })
  418. expect(fake.removed).toContain('/workspace/.dsh-e2b/processes/one/stderr.log')
  419. await expect(handle.waitForExit()).resolves.toBe(true)
  420. })
  421. it('rejects an unrepresentable graceMs before any remote work', () => {
  422. const ctx = new Context()
  423. const service = Object.create(E2BSubprocessRuntime.prototype) as E2BSubprocessRuntime
  424. Reflect.set(service, 'disposing', false)
  425. Reflect.set(service, 'ctx', ctx)
  426. for (const graceMs of [0, -1, Number.NaN, Number.POSITIVE_INFINITY]) {
  427. expect(() => service.spawn(spec({ graceMs }))).toThrow('graceMs must be a positive finite number')
  428. void expect(service.spawnTerminal({
  429. argv: ['bash'], cwd: '/w', rows: 24, cols: 80, graceMs,
  430. })).rejects.toThrow('graceMs must be a positive finite number')
  431. }
  432. })
  433. it('rejects malformed environment entries before command start', async () => {
  434. for (const env of [{ 'BAD=NAME': 'x' }, { BAD: 'x\0INJECTED=1' }]) {
  435. const fake = new FakeSandbox()
  436. const handle = testHandle(runtime(fake), spec({ env }), '/runtime/invalid-environment')
  437. await expect(handle.done).rejects.toThrow('environment entries')
  438. expect(fake.startOptions).toBeUndefined()
  439. expect(fake.removed).toContain('/runtime/invalid-environment')
  440. }
  441. })
  442. it('preserves UTF-8 bytes when the ASCII transport is split across callbacks', async () => {
  443. const fake = new FakeSandbox()
  444. const handle = testHandle(runtime(fake), spec({
  445. stdio: { stdin: 'ignore', stdout: 'pipe', stderr: { maxBytes: 4 } },
  446. }), '/runtime/split-utf8')
  447. await flush()
  448. const chunks: Buffer[] = []
  449. handle.stdout!.on('data', (chunk: Buffer) => { chunks.push(chunk) })
  450. for (const character of `${Buffer.from('A你好B').toString('base64')}\n`) {
  451. await fake.stdoutWire(character)
  452. }
  453. fake.finish()
  454. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  455. expect(Buffer.concat(chunks).toString('utf8')).toBe('A你好B')
  456. })
  457. it('rejects malformed output transport without confusing it with a consumer sink failure', async () => {
  458. const fake = new FakeSandbox()
  459. const handle = testHandle(runtime(fake), spec(), '/runtime/malformed-output')
  460. await flush()
  461. await fake.stdoutWire('%\n')
  462. fake.finish()
  463. await expect(handle.done).rejects.toThrow('invalid base64 output transport')
  464. const stderrFake = new FakeSandbox()
  465. const stderrHandle = testHandle(runtime(stderrFake), spec(), '/runtime/malformed-stderr')
  466. await flush()
  467. await stderrFake.stderrWire('%\n')
  468. stderrFake.finish()
  469. await expect(stderrHandle.done).rejects.toThrow('invalid base64 output transport')
  470. })
  471. it('rejects a naturally completed command whose encoder omits its completion frame', async () => {
  472. const fake = new FakeSandbox()
  473. const handle = testHandle(runtime(fake), spec(), '/runtime/incomplete-output')
  474. await flush()
  475. fake.alive = false
  476. fake.handle.succeed(0)
  477. await expect(handle.done).rejects.toThrow('incomplete output transport')
  478. })
  479. it('bounds descendant-held output draining and withholds the incomplete spill', async () => {
  480. const fake = new FakeSandbox()
  481. const handle = testHandle(runtime(fake), spec({ graceMs: 5 }), '/runtime/drain-bound')
  482. await flush()
  483. await fake.stdout('leader-output')
  484. fake.exitStatus = '0\n'
  485. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  486. expect(fake.handle.disconnects).toBe(1)
  487. expect(handle.collected.stdout?.readFrom(0)).toEqual({
  488. text: 'tput',
  489. nextOffset: 13,
  490. lossy: true,
  491. })
  492. expect(fake.removed).toContain('/runtime/drain-bound/stdout.log')
  493. handle.terminate()
  494. await expect(handle.waitForExit()).resolves.toBe(true)
  495. })
  496. it('releases an inherited-output callback blocked on host backpressure at drain expiry', async () => {
  497. const fake = new FakeSandbox()
  498. const written: string[] = []
  499. const stdoutWrite = vi.spyOn(process.stdout, 'write').mockImplementation(((chunk: Uint8Array) => {
  500. written.push(Buffer.from(chunk).toString())
  501. return false
  502. }) as typeof process.stdout.write)
  503. try {
  504. const handle = testHandle(runtime(fake), spec({
  505. graceMs: 5,
  506. stdio: { stdin: 'ignore', stdout: 'inherit', stderr: { maxBytes: 4 } },
  507. }), '/runtime/inherit-backpressure')
  508. await flush()
  509. let callbackSettled = false
  510. const blocked = fake.stdout('blocked bytes').then(() => { callbackSettled = true })
  511. await flush()
  512. expect(callbackSettled).toBe(false)
  513. fake.exitStatus = '0\n'
  514. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  515. await blocked
  516. expect(callbackSettled).toBe(true)
  517. expect(written.join('')).toBe('blocked bytes')
  518. expect(fake.handle.disconnects).toBe(1)
  519. handle.terminate()
  520. await expect(handle.waitForExit()).resolves.toBe(true)
  521. } finally {
  522. stdoutWrite.mockRestore()
  523. }
  524. })
  525. it('waits for lossless raw-pipe output after the direct status is published', async () => {
  526. const fake = new FakeSandbox()
  527. const handle = testHandle(runtime(fake), spec({
  528. graceMs: 1,
  529. stdio: { stdin: 'ignore', stdout: 'pipe', stderr: { maxBytes: 4 } },
  530. }), '/runtime/pipe-drain')
  531. let output = ''
  532. handle.stdout!.on('data', (chunk) => { output += String(chunk) })
  533. await flush()
  534. fake.exitStatus = '0\n'
  535. let settled = false
  536. void handle.done.then(() => { settled = true })
  537. await new Promise(resolve => setTimeout(resolve, 50))
  538. expect(settled).toBe(false)
  539. expect(fake.handle.disconnects).toBe(0)
  540. expect(fake.statusReads).toBe(0)
  541. await fake.stdout('complete protocol frame')
  542. fake.finish()
  543. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  544. expect(output).toBe('complete protocol frame')
  545. expect(fake.statusReads).toBe(1)
  546. })
  547. it('accepts clean encoder completion inside the output-drain grace', async () => {
  548. const fake = new FakeSandbox()
  549. const handle = testHandle(runtime(fake), spec({ graceMs: 100 }), '/runtime/drain-complete')
  550. await flush()
  551. fake.exitStatus = '0\n'
  552. fake.afterStatusRead = () => {
  553. fake.afterStatusRead = undefined
  554. setTimeout(() => { fake.finish() }, 0)
  555. }
  556. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  557. expect(fake.handle.disconnects).toBe(0)
  558. })
  559. it('preserves a published exit code when requested termination outlives output draining', async () => {
  560. const fake = new FakeSandbox()
  561. fake.trapsTerm = true
  562. fake.delaysKill = true
  563. fake.delaysKillCompletion = true
  564. fake.sdkKillStops = false
  565. const handle = testHandle(runtime(fake), spec({ graceMs: 5 }), '/runtime/drain-signal')
  566. await flush()
  567. handle.terminate()
  568. await vi.waitFor(() => { expect(fake.commandsSeen).toContain('kill -KILL -- -4242') })
  569. fake.exitStatus = '0\n'
  570. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  571. expect(fake.handle.disconnects).toBe(1)
  572. fake.alive = false
  573. handle.terminate()
  574. await expect(handle.waitForExit()).resolves.toBe(true)
  575. })
  576. it('preserves a published nonzero exit code when termination settles the SDK inside the drain grace', async () => {
  577. const fake = new FakeSandbox()
  578. const handle = testHandle(runtime(fake), spec({ graceMs: 100 }), '/runtime/drain-signal-settled')
  579. await flush()
  580. fake.exitStatus = '7\n'
  581. fake.afterStatusRead = () => {
  582. fake.afterStatusRead = undefined
  583. handle.terminate()
  584. }
  585. await expect(handle.done).resolves.toEqual({ exitCode: 7, signal: null })
  586. await expect(handle.waitForExit()).resolves.toBe(true)
  587. })
  588. it('rejects an invalid direct-command exit status', async () => {
  589. const fake = new FakeSandbox()
  590. const handle = testHandle(runtime(fake), spec(), '/runtime/invalid-status')
  591. await flush()
  592. fake.exitStatus = '999\n'
  593. await expect(handle.done).rejects.toThrow('invalid exit code')
  594. handle.terminate()
  595. await expect(handle.waitForExit()).resolves.toBe(true)
  596. })
  597. it('rolls back a published process group before rejecting a monitoring failure', async () => {
  598. const fake = new FakeSandbox()
  599. fake.statusError = new Error('status transport failed')
  600. const handle = testHandle(runtime(fake), spec(), '/runtime/status-failure')
  601. await expect(handle.done).rejects.toThrow('status transport failed')
  602. expect(fake.commandsSeen).toContain('kill -TERM -- -4242')
  603. expect(fake.alive).toBe(false)
  604. await expect(handle.waitForExit()).resolves.toBe(true)
  605. const failed = new FakeSandbox()
  606. failed.statusError = new Error('status transport failed')
  607. failed.signalErrors.push(new Error('TERM transport failed'), new Error('KILL transport failed'))
  608. failed.handle.killError = new Error('SDK kill failed')
  609. const retained = testHandle(runtime(failed), spec({ graceMs: 1 }), '/runtime/status-cleanup-failure')
  610. await expect(retained.done).rejects.toThrow(
  611. 'command monitoring failed and process-group rollback did not reach quiescence',
  612. )
  613. expect(failed.alive).toBe(true)
  614. failed.handle.killError = undefined
  615. retained.terminate()
  616. await expect(retained.waitForExit()).resolves.toBe(true)
  617. // A state-cleanup failure on top preserves the rollback failure instead of
  618. // re-aggregating only the original monitoring error.
  619. const triple = new FakeSandbox()
  620. triple.statusError = new Error('status transport failed')
  621. triple.signalErrors.push(new Error('TERM transport failed'), new Error('KILL transport failed'))
  622. triple.handle.killError = new Error('SDK kill failed')
  623. triple.nextRemoveError = new Error('state cleanup failed')
  624. const tripleHandle = testHandle(runtime(triple), spec({ graceMs: 1 }), '/runtime/triple-failure')
  625. const failure = await tripleHandle.done.catch((error: unknown) => error as AggregateError)
  626. expect(failure).toBeInstanceOf(AggregateError)
  627. expect((failure as AggregateError).message).toContain('private state cleanup failed')
  628. const nested = (failure as AggregateError).errors[0] as AggregateError
  629. expect(nested.message).toContain('rollback did not reach quiescence')
  630. triple.handle.killError = undefined
  631. tripleHandle.terminate()
  632. await expect(tripleHandle.waitForExit()).resolves.toBe(true)
  633. })
  634. it('surfaces deferred piped-stdin write and close failures as stream errors', async () => {
  635. const writeFake = new FakeSandbox()
  636. writeFake.deferStart()
  637. vi.spyOn(writeFake.handle, 'sendStdin').mockRejectedValueOnce('stdin rejected')
  638. const writeHandle = testHandle(runtime(writeFake), spec({
  639. stdio: { stdin: 'pipe', stdout: { maxBytes: 4 }, stderr: { maxBytes: 4 } },
  640. }), '/runtime/stdin-write-error')
  641. const writeError = once(writeHandle.stdin!, 'error')
  642. writeHandle.stdin!.write('input')
  643. writeFake.releaseStart()
  644. await expect(writeError).resolves.toMatchObject([{ message: 'stdin rejected' }])
  645. writeFake.finish()
  646. await writeHandle.done
  647. const closeFake = new FakeSandbox()
  648. vi.spyOn(closeFake.handle, 'closeStdin').mockRejectedValueOnce(new Error('close rejected'))
  649. const closeHandle = testHandle(runtime(closeFake), spec({
  650. stdio: { stdin: 'pipe', stdout: { maxBytes: 4 }, stderr: { maxBytes: 4 } },
  651. }), '/runtime/stdin-close-error')
  652. await flush()
  653. const closeError = once(closeHandle.stdin!, 'error')
  654. closeHandle.stdin!.end()
  655. await expect(closeError).resolves.toMatchObject([{ message: 'close rejected' }])
  656. closeFake.finish()
  657. await closeHandle.done
  658. })
  659. it('collects bounded tails, retains valid spills, and maps natural nonzero exits', async () => {
  660. const fake = new FakeSandbox()
  661. const handle = testHandle(runtime(fake), spec({
  662. stdio: {
  663. stdin: { data: 'batch' },
  664. stdout: { maxBytes: 4, spill: { maxBytes: 16 } },
  665. stderr: { maxBytes: 3 },
  666. },
  667. }), '/runtime/two')
  668. await flush()
  669. await fake.stdout('abcdef')
  670. await fake.stderr('12345')
  671. fake.finish(7)
  672. await expect(handle.done).resolves.toEqual({ exitCode: 7, signal: null })
  673. expect(fake.handle.sent).toEqual(['batch'])
  674. expect(fake.handle.closes).toBe(1)
  675. expect(handle.collected.stdout!.readFrom(0)).toEqual({
  676. text: 'cdef',
  677. nextOffset: 6,
  678. lossy: true,
  679. spillPath: '/runtime/two/stdout.log',
  680. })
  681. expect(handle.collected.stderr!.readFrom(0)).toEqual({ text: '345', nextOffset: 5, lossy: true })
  682. expect(fake.removed).not.toContain('/runtime/two/stdout.log')
  683. })
  684. it('removes a spill once the complete stream exceeds its cap', async () => {
  685. const fake = new FakeSandbox()
  686. const handle = testHandle(runtime(fake), spec({
  687. stdio: { stdin: 'ignore', stdout: { maxBytes: 2, spill: { maxBytes: 3 } }, stderr: 'inherit' },
  688. }), '/runtime/oversize')
  689. await flush()
  690. await fake.stdout('abcd')
  691. await fake.stderr('')
  692. fake.finish()
  693. await handle.done
  694. expect(handle.collected.stdout!.readFrom(0)).toEqual({ text: 'cd', nextOffset: 4, lossy: true })
  695. expect(fake.removed).toContain('/runtime/oversize/stdout.log')
  696. const command = fake.commandsSeen.find(value => value.includes('dsh_e2b_tee='))!
  697. expect(command).toContain('"$dsh_e2b_head" -c 3')
  698. expect(command).toContain('/runtime/oversize/stdout.log')
  699. expect(command).toContain('"$dsh_e2b_tee" --output-error=warn-nopipe')
  700. expect(command).not.toContain('tee -a')
  701. })
  702. it('contains remote spill-removal failures and routes empty inherited output', async () => {
  703. const fake = new FakeSandbox()
  704. fake.nextRemoveError = new Error('already removed')
  705. const handle = testHandle(runtime(fake), spec({
  706. stdio: { stdin: 'ignore', stdout: 'inherit', stderr: { maxBytes: 4, spill: { maxBytes: 8 } } },
  707. }), '/runtime/remove-error')
  708. await flush()
  709. await fake.stdout('')
  710. await fake.stderr('')
  711. fake.finish()
  712. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  713. expect(fake.removed).toContain('/runtime/remove-error/stderr.log')
  714. })
  715. it('terminates a process group with TERM and reports the signal outcome', async () => {
  716. const fake = new FakeSandbox()
  717. const handle = testHandle(runtime(fake), spec(), '/runtime/term')
  718. await flush()
  719. handle.terminate()
  720. handle.terminate()
  721. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGTERM' })
  722. await expect(handle.waitForExit()).resolves.toBe(true)
  723. expect(fake.commandsSeen).toContain('kill -TERM -- -4242')
  724. expect(fake.commandsSeen).not.toContain('kill -KILL -- -4242')
  725. const signals = fake.commandsSeen.filter(command => command.startsWith('kill -')).length
  726. fake.alive = true
  727. handle.terminate()
  728. await flush()
  729. expect(fake.alive).toBe(true)
  730. expect(fake.commandsSeen.filter(command => command.startsWith('kill -'))).toHaveLength(signals)
  731. })
  732. it('makes termination a permanent no-op after natural quiescence is observed', async () => {
  733. const fake = new FakeSandbox()
  734. const handle = testHandle(runtime(fake), spec(), '/runtime/natural-quiescence')
  735. await flush()
  736. fake.finish()
  737. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  738. await expect(handle.waitForExit()).resolves.toBe(true)
  739. const signals = fake.commandsSeen.filter(command => command.startsWith('kill -')).length
  740. fake.alive = true
  741. handle.terminate()
  742. await flush()
  743. expect(fake.alive).toBe(true)
  744. expect(fake.commandsSeen.filter(command => command.startsWith('kill -'))).toHaveLength(signals)
  745. })
  746. it('treats a zombie-only process group as quiescent', async () => {
  747. const fake = new FakeSandbox()
  748. fake.zombieOnly = true
  749. const handle = testHandle(runtime(fake), spec(), '/runtime/zombie-quiescence')
  750. await flush()
  751. await expect(handle.waitForExit()).resolves.toBe(true)
  752. expect(fake.commandsSeen).toContain(
  753. 'set -o pipefail; ps -eo pgid=,stat= | awk \'$1 == 4242 && $2 !~ /^[ZXx]/ { live=1 } END { if (live) print "live" }\'',
  754. )
  755. fake.finish()
  756. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  757. })
  758. it('keeps proven quiescence after a concurrent termination transport fails', async () => {
  759. const fake = new FakeSandbox()
  760. fake.signalErrors.push(new Error('TERM transport failed'), new Error('KILL transport failed'))
  761. fake.handle.killError = new Error('SDK kill failed')
  762. fake.deferSignals()
  763. const handle = testHandle(runtime(fake), spec(), '/runtime/quiescent-race')
  764. await flush()
  765. handle.terminate()
  766. await vi.waitFor(() => { expect(fake.commandsSeen).toContain('kill -TERM -- -4242') })
  767. fake.alive = false
  768. await expect(handle.waitForExit()).resolves.toBe(true)
  769. fake.probeError = new Error('post-quiescence probe failed')
  770. fake.releaseSignals()
  771. await vi.waitFor(() => { expect(fake.handle.kills).toBe(1) })
  772. fake.finish()
  773. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  774. const signals = fake.commandsSeen.filter(command => command.startsWith('kill -')).length
  775. handle.terminate()
  776. await expect(handle.waitForExit()).resolves.toBe(true)
  777. expect(fake.commandsSeen.filter(command => command.startsWith('kill -'))).toHaveLength(signals)
  778. })
  779. it('escalates a TERM-trapping process group to KILL and uses the SDK kill as fallback', async () => {
  780. const fake = new FakeSandbox()
  781. fake.trapsTerm = true
  782. fake.handle.killError = new Error('already gone')
  783. const handle = testHandle(runtime(fake), spec({ graceMs: 1 }), '/runtime/kill')
  784. await flush()
  785. handle.terminate()
  786. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGKILL' })
  787. await expect(handle.waitForExit()).resolves.toBe(true)
  788. expect(fake.commandsSeen).toContain('kill -KILL -- -4242')
  789. expect(fake.handle.kills).toBe(1)
  790. })
  791. it('keeps force cleanup retryable until quiescence is proven', async () => {
  792. const fake = new FakeSandbox()
  793. fake.trapsTerm = true
  794. fake.delaysKill = true
  795. fake.delaysKillCompletion = true
  796. fake.sdkKillStops = false
  797. const handle = testHandle(runtime(fake), spec({ graceMs: 1 }), '/runtime/termination-fence')
  798. await flush()
  799. handle.terminate()
  800. await vi.waitFor(() => { expect(fake.handle.kills).toBe(1) })
  801. await expect(handle.waitForExit()).rejects.toThrow('remained live after force termination')
  802. expect(fake.alive).toBe(true)
  803. fake.delaysKill = false
  804. fake.delaysKillCompletion = false
  805. fake.sdkKillStops = true
  806. handle.terminate()
  807. await expect(handle.waitForExit()).resolves.toBe(true)
  808. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGKILL' })
  809. })
  810. it('honors termination requested before asynchronous startup finishes', async () => {
  811. const fake = new FakeSandbox()
  812. fake.deferStart()
  813. const handle = testHandle(runtime(fake), spec(), '/runtime/deferred-kill')
  814. handle.terminate()
  815. fake.releaseStart()
  816. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGTERM' })
  817. })
  818. it('aborts a stalled preparation request before reporting startup quiescence', async () => {
  819. const fake = new FakeSandbox()
  820. let preparationSignal: AbortSignal | undefined
  821. fake.environmentRequest = async (signal) => {
  822. preparationSignal = signal
  823. await new Promise<never>((_resolve, reject) => {
  824. const rejectAbort = (): void => {
  825. const reason: unknown = signal?.reason
  826. reject(reason instanceof Error ? reason : new Error(String(reason)))
  827. }
  828. if (signal?.aborted === true) {
  829. rejectAbort()
  830. return
  831. }
  832. signal?.addEventListener('abort', rejectAbort, { once: true })
  833. })
  834. }
  835. const handle = testHandle(runtime(fake), spec(), '/runtime/stalled-preparation')
  836. await vi.waitFor(() => { expect(preparationSignal).toBeDefined() })
  837. handle.terminate()
  838. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGTERM' })
  839. await expect(handle.waitForExit()).resolves.toBe(true)
  840. expect(preparationSignal?.aborted).toBe(true)
  841. expect(fake.startOptions).toBeUndefined()
  842. })
  843. it('kills through the provisional SDK handle before process-group publication', async () => {
  844. const fake = new FakeSandbox()
  845. fake.deferProcessGroupRead()
  846. fake.signalErrors.push(commandError(1), commandError(1))
  847. const handle = testHandle(runtime(fake), spec(), '/runtime/pre-publication-kill')
  848. await vi.waitFor(() => { expect(fake.startOptions).toBeDefined() })
  849. handle.terminate()
  850. await vi.waitFor(() => { expect(fake.handle.kills).toBe(1) })
  851. expect(fake.alive).toBe(false)
  852. await expect(handle.waitForExit()).resolves.toBe(true)
  853. fake.releaseProcessGroupRead()
  854. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGKILL' })
  855. })
  856. it('does not treat an unsuccessful SDK fallback as provisional group quiescence', async () => {
  857. const fake = new FakeSandbox()
  858. fake.deferProcessGroupRead()
  859. fake.trapsTerm = true
  860. fake.delaysKill = true
  861. fake.delaysKillCompletion = true
  862. fake.sdkKillStops = false
  863. fake.handle.killResult = false
  864. const handle = testHandle(runtime(fake), spec({ graceMs: 1 }), '/runtime/provisional-sdk-false')
  865. await vi.waitFor(() => { expect(fake.startOptions).toBeDefined() })
  866. handle.terminate()
  867. await vi.waitFor(() => { expect(fake.handle.kills).toBe(1) })
  868. await expect(handle.waitForExit()).rejects.toThrow('remained live after force termination')
  869. fake.alive = false
  870. handle.terminate()
  871. await expect(handle.waitForExit()).resolves.toBe(true)
  872. fake.releaseProcessGroupRead()
  873. fake.finish()
  874. await handle.done
  875. })
  876. it('bounds a quiescence observer while provisional termination is awaiting the controller', async () => {
  877. const fake = new FakeSandbox()
  878. fake.deferProcessGroupRead()
  879. const reconnect = Promise.withResolvers<Sandbox>()
  880. let calls = 0
  881. const delayedRuntime = runtime(fake, async () => {
  882. calls += 1
  883. return calls === 1 ? fake.sandbox : await reconnect.promise
  884. })
  885. const handle = testHandle(delayedRuntime, spec(), '/runtime/pre-publication-observer')
  886. await vi.waitFor(() => { expect(fake.startOptions).toBeDefined() })
  887. handle.terminate()
  888. const controller = new AbortController()
  889. const waiting = handle.waitForExit(controller.signal)
  890. await flush()
  891. controller.abort()
  892. await expect(waiting).resolves.toBe(false)
  893. reconnect.resolve(fake.sandbox)
  894. await expect(handle.waitForExit()).resolves.toBe(true)
  895. fake.releaseProcessGroupRead()
  896. await handle.done
  897. })
  898. it('proves a provisional group exit when the SDK kill fallback fails', async () => {
  899. const fake = new FakeSandbox()
  900. fake.deferProcessGroupRead()
  901. fake.trapsTerm = true
  902. fake.handle.killError = new Error('SDK kill unavailable')
  903. const handle = testHandle(runtime(fake), spec({ graceMs: 1 }), '/runtime/pre-publication-group-kill')
  904. await vi.waitFor(() => { expect(fake.startOptions).toBeDefined() })
  905. handle.terminate()
  906. await expect(handle.waitForExit()).resolves.toBe(true)
  907. fake.releaseProcessGroupRead()
  908. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGKILL' })
  909. })
  910. it('reports failed provisional group and SDK force transports', async () => {
  911. const fake = new FakeSandbox()
  912. fake.deferProcessGroupRead()
  913. fake.signalErrors.push(new Error('TERM transport failed'), new Error('KILL transport failed'))
  914. fake.handle.killError = new Error('SDK kill failed')
  915. const handle = testHandle(runtime(fake), spec({ graceMs: 1 }), '/runtime/pre-publication-failure')
  916. await vi.waitFor(() => { expect(fake.startOptions).toBeDefined() })
  917. handle.terminate()
  918. await expect(handle.waitForExit()).rejects.toThrow('remained live after force termination')
  919. fake.handle.killError = undefined
  920. handle.terminate()
  921. await expect(handle.waitForExit()).resolves.toBe(true)
  922. fake.releaseProcessGroupRead()
  923. await handle.done
  924. const absentGroup = new FakeSandbox()
  925. absentGroup.deferProcessGroupRead()
  926. absentGroup.signalErrors.push(commandError(1), commandError(1))
  927. absentGroup.handle.killError = new Error('SDK kill failed without a provisional group')
  928. const absentHandle = testHandle(
  929. runtime(absentGroup),
  930. spec({ graceMs: 1 }),
  931. '/runtime/pre-publication-absent-group',
  932. )
  933. await vi.waitFor(() => { expect(absentGroup.startOptions).toBeDefined() })
  934. absentHandle.terminate()
  935. await expect(absentHandle.waitForExit()).rejects.toThrow('remained live after force termination')
  936. absentGroup.handle.killError = undefined
  937. absentHandle.terminate()
  938. await expect(absentHandle.waitForExit()).resolves.toBe(true)
  939. absentGroup.releaseProcessGroupRead()
  940. await absentHandle.done
  941. const optimisticSdk = new FakeSandbox()
  942. optimisticSdk.deferProcessGroupRead()
  943. optimisticSdk.signalErrors.push(commandError(1), commandError(1))
  944. optimisticSdk.sdkKillStops = false
  945. const optimisticHandle = testHandle(
  946. runtime(optimisticSdk),
  947. spec({ graceMs: 1 }),
  948. '/runtime/pre-publication-optimistic-sdk',
  949. )
  950. await vi.waitFor(() => { expect(optimisticSdk.startOptions).toBeDefined() })
  951. optimisticHandle.terminate()
  952. await expect(optimisticHandle.waitForExit()).rejects.toThrow('remained live after force termination')
  953. optimisticHandle.terminate()
  954. await expect(optimisticHandle.waitForExit()).resolves.toBe(true)
  955. optimisticSdk.releaseProcessGroupRead()
  956. await optimisticHandle.done
  957. })
  958. it('honors an already-aborted signal when constructing the asynchronous handle directly', async () => {
  959. const fake = new FakeSandbox()
  960. const handle = testHandle(runtime(fake), spec({ signal: AbortSignal.abort('stop') }), '/runtime/pre-aborted')
  961. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGTERM' })
  962. })
  963. it('reacts to a signal that aborts after the remote command has started', async () => {
  964. const fake = new FakeSandbox()
  965. const controller = new AbortController()
  966. const handle = testHandle(runtime(fake), spec({ signal: controller.signal }), '/runtime/live-abort')
  967. await flush()
  968. controller.abort('stop')
  969. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGTERM' })
  970. })
  971. it('can terminate a surviving process group after the command leader settles', async () => {
  972. const fake = new FakeSandbox()
  973. const handle = testHandle(runtime(fake), spec(), '/runtime/surviving-group')
  974. await flush()
  975. await fake.completeOutput()
  976. fake.handle.succeed(0)
  977. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  978. expect(fake.alive).toBe(true)
  979. handle.terminate()
  980. await flush()
  981. const signaled = fake.commandsSeen.includes('kill -TERM -- -4242')
  982. if (!signaled) fake.finish()
  983. await expect(handle.waitForExit()).resolves.toBe(true)
  984. expect(signaled).toBe(true)
  985. })
  986. it('bounds waitForExit while startup or a live group is pending', async () => {
  987. const fake = new FakeSandbox()
  988. fake.deferStart()
  989. const handle = testHandle(runtime(fake), spec(), '/runtime/wait')
  990. const beforeStart = new AbortController()
  991. const pending = handle.waitForExit(beforeStart.signal)
  992. beforeStart.abort()
  993. await expect(pending).resolves.toBe(false)
  994. await expect(handle.waitForExit(AbortSignal.abort())).resolves.toBe(false)
  995. fake.releaseStart()
  996. await flush()
  997. const live = new AbortController()
  998. const liveWait = handle.waitForExit(live.signal)
  999. live.abort()
  1000. await expect(liveWait).resolves.toBe(false)
  1001. fake.finish()
  1002. await handle.done
  1003. const terminatingFake = new FakeSandbox()
  1004. terminatingFake.deferStart()
  1005. const terminating = testHandle(runtime(terminatingFake), spec(), '/runtime/wait-termination-start')
  1006. terminating.terminate()
  1007. const beforeHandle = new AbortController()
  1008. const handlePending = terminating.waitForExit(beforeHandle.signal)
  1009. beforeHandle.abort()
  1010. await expect(handlePending).resolves.toBe(false)
  1011. terminatingFake.releaseStart()
  1012. await terminating.done
  1013. })
  1014. it('bounds both sides of the liveness-poll abort race', async () => {
  1015. const fake = new FakeSandbox()
  1016. const handle = testHandle(runtime(fake), spec(), '/runtime/poll-abort')
  1017. await flush()
  1018. const beforeTick = new AbortController()
  1019. fake.afterProbe = () => { beforeTick.abort(); fake.afterProbe = undefined }
  1020. await expect(handle.waitForExit(beforeTick.signal)).resolves.toBe(false)
  1021. const duringTick = new AbortController()
  1022. fake.afterProbe = () => {
  1023. fake.afterProbe = undefined
  1024. setTimeout(() => { duringTick.abort() }, 0)
  1025. }
  1026. await expect(handle.waitForExit(duringTick.signal)).resolves.toBe(false)
  1027. const duringProbe = new AbortController()
  1028. fake.beforeProbe = () => { duringProbe.abort(); fake.beforeProbe = undefined }
  1029. await expect(handle.waitForExit(duringProbe.signal)).resolves.toBe(false)
  1030. let racedAbort = false
  1031. const raceSignal = {
  1032. get aborted() { return racedAbort },
  1033. addEventListener: () => { racedAbort = true },
  1034. removeEventListener: () => {},
  1035. } as unknown as AbortSignal
  1036. await expect(handle.waitForExit(raceSignal)).resolves.toBe(false)
  1037. fake.finish()
  1038. await handle.done
  1039. })
  1040. it('observes a live group across one successful bounded poll', async () => {
  1041. const fake = new FakeSandbox()
  1042. const handle = testHandle(runtime(fake), spec(), '/runtime/poll-success')
  1043. await flush()
  1044. setTimeout(() => { fake.finish() }, 1)
  1045. await expect(handle.waitForExit(new AbortController().signal)).resolves.toBe(true)
  1046. await handle.done
  1047. })
  1048. it('treats startup failure as no live tree and contains readiness rejection', async () => {
  1049. const fake = new FakeSandbox()
  1050. fake.backgroundError = new Error('start failed')
  1051. const handle = testHandle(runtime(fake), spec(), '/runtime/fail')
  1052. await expect(handle.done).rejects.toThrow('start failed')
  1053. expect(fake.removed).toContain('/runtime/fail/environment')
  1054. expect(fake.removed).toContain('/runtime/fail')
  1055. await expect(handle.waitForExit()).resolves.toBe(true)
  1056. handle.terminate()
  1057. const unavailableHandle = testHandle(
  1058. runtime(new FakeSandbox(), async () => { throw new Error('sandbox unavailable') }),
  1059. spec(),
  1060. '/runtime/unavailable-start',
  1061. )
  1062. await expect(unavailableHandle.done).rejects.toThrow('sandbox unavailable')
  1063. await expect(unavailableHandle.waitForExit()).resolves.toBe(true)
  1064. const envFailure = new FakeSandbox()
  1065. envFailure.envError = new Error('ambient lookup failed')
  1066. const envHandle = testHandle(runtime(envFailure), spec(), '/runtime/env-failure')
  1067. await expect(envHandle.done).rejects.toThrow('ambient lookup failed')
  1068. expect(envFailure.removed).toEqual([])
  1069. const expectEnvironmentFailure = async (name: string, wire: string, message: string): Promise<void> => {
  1070. const fake = new FakeSandbox()
  1071. fake.environmentWire = wire
  1072. const failed = testHandle(runtime(fake), spec(), `/runtime/${name}`)
  1073. await expect(failed.done).rejects.toThrow(message)
  1074. }
  1075. const encodedEnvironment = Buffer.from('PATH=/bin\0').toString('base64')
  1076. const encodedHome = Buffer.from('/home/user').toString('base64')
  1077. await expectEnvironmentFailure('malformed-frame', '%', 'invalid base64')
  1078. await expectEnvironmentFailure('malformed-base64', `${encodedHome}\n%`, 'invalid base64')
  1079. await expectEnvironmentFailure(
  1080. 'invalid-utf8-home',
  1081. `${Buffer.from([0xff]).toString('base64')}\n${encodedEnvironment}`,
  1082. 'not valid UTF-8',
  1083. )
  1084. await expectEnvironmentFailure(
  1085. 'invalid-utf8-environment',
  1086. `${encodedHome}\n${Buffer.from([0xff]).toString('base64')}`,
  1087. 'not valid UTF-8',
  1088. )
  1089. await expectEnvironmentFailure(
  1090. 'relative-home',
  1091. `${Buffer.from('home/user').toString('base64')}\n${encodedEnvironment}`,
  1092. 'remote login home is invalid',
  1093. )
  1094. await expectEnvironmentFailure(
  1095. 'nul-home',
  1096. `${Buffer.from('/home/user\0tail').toString('base64')}\n${encodedEnvironment}`,
  1097. 'remote login home is invalid',
  1098. )
  1099. const cleanupFailure = new FakeSandbox()
  1100. cleanupFailure.backgroundError = new Error('start failed before credential consumption')
  1101. cleanupFailure.nextRemoveError = new Error('credential cleanup failed')
  1102. const cleanupHandle = testHandle(runtime(cleanupFailure), spec(), '/runtime/cleanup-failure')
  1103. await expect(cleanupHandle.done).rejects.toThrow('command failed and private state cleanup failed')
  1104. const absentState = new FakeSandbox()
  1105. absentState.backgroundError = new Error('start failed after external cleanup')
  1106. absentState.nextRemoveError = new FileNotFoundError('already removed')
  1107. const absentHandle = testHandle(runtime(absentState), spec(), '/runtime/absent-state')
  1108. await expect(absentHandle.done).rejects.toThrow('start failed after external cleanup')
  1109. })
  1110. it('bounds a readiness rejection with a still-live caller signal', async () => {
  1111. const fake = new FakeSandbox()
  1112. fake.deferStart()
  1113. fake.backgroundError = new Error('start failed')
  1114. const handle = testHandle(runtime(fake), spec(), '/runtime/fail-with-signal')
  1115. const waiting = handle.waitForExit(new AbortController().signal)
  1116. fake.releaseStart()
  1117. await expect(handle.done).rejects.toThrow('start failed')
  1118. await expect(waiting).resolves.toBe(true)
  1119. })
  1120. it('propagates an unavailable sandbox unless the caller aborts the wait', async () => {
  1121. const fake = new FakeSandbox()
  1122. let calls = 0
  1123. const unavailable = runtime(fake, async () => {
  1124. calls += 1
  1125. if (calls === 1) return fake.sandbox
  1126. throw new Error('connection unavailable')
  1127. })
  1128. const handle = testHandle(unavailable, spec(), '/runtime/unavailable')
  1129. await flush()
  1130. await expect(handle.waitForExit()).rejects.toThrow('connection unavailable')
  1131. fake.finish()
  1132. await handle.done
  1133. })
  1134. it('returns false when the caller aborts while reconnecting for liveness', async () => {
  1135. const fake = new FakeSandbox()
  1136. const reconnect = Promise.withResolvers<Sandbox>()
  1137. let calls = 0
  1138. const unavailable = runtime(fake, async () => {
  1139. calls += 1
  1140. return calls === 1 ? fake.sandbox : await reconnect.promise
  1141. })
  1142. const handle = testHandle(unavailable, spec(), '/runtime/reconnect-abort')
  1143. await flush()
  1144. const controller = new AbortController()
  1145. const waiting = handle.waitForExit(controller.signal)
  1146. await flush()
  1147. controller.abort()
  1148. reconnect.reject(new Error('connection unavailable'))
  1149. await expect(waiting).resolves.toBe(false)
  1150. fake.finish()
  1151. await handle.done
  1152. })
  1153. it('returns false when a liveness request itself is aborted and surfaces other probe failures', async () => {
  1154. const fake = new FakeSandbox()
  1155. const handle = testHandle(runtime(fake), spec(), '/runtime/probe')
  1156. await flush()
  1157. const controller = new AbortController()
  1158. controller.abort()
  1159. await expect(handle.waitForExit(controller.signal)).resolves.toBe(false)
  1160. fake.probeError = new Error('probe failed')
  1161. await expect(handle.waitForExit()).rejects.toThrow('probe failed')
  1162. fake.finish()
  1163. await handle.done
  1164. })
  1165. it('treats a timeout-killed sandbox as quiescent during liveness probing', async () => {
  1166. const fake = new FakeSandbox()
  1167. const handle = testHandle(runtime(fake), spec(), '/runtime/expired-sandbox')
  1168. await flush()
  1169. fake.finish()
  1170. await handle.done
  1171. fake.probeError = new SandboxNotFoundError('sandbox expired')
  1172. await expect(handle.waitForExit()).resolves.toBe(true)
  1173. })
  1174. it('treats a missing sandbox handle as quiescent during liveness acquisition', async () => {
  1175. const fake = new FakeSandbox()
  1176. let calls = 0
  1177. const handle = testHandle(runtime(fake, async () => {
  1178. calls += 1
  1179. if (calls === 1) return fake.sandbox
  1180. throw new SandboxNotFoundError('sandbox expired')
  1181. }), spec(), '/runtime/expired-acquisition')
  1182. await flush()
  1183. await expect(handle.waitForExit()).resolves.toBe(true)
  1184. await fake.completeOutput()
  1185. fake.alive = false
  1186. fake.handle.succeed(0)
  1187. await handle.done
  1188. })
  1189. it('treats sandbox loss during termination as quiescent', async () => {
  1190. const fake = new FakeSandbox()
  1191. let calls = 0
  1192. const handle = testHandle(runtime(fake, async () => {
  1193. calls += 1
  1194. if (calls === 1) return fake.sandbox
  1195. throw new SandboxNotFoundError('sandbox expired')
  1196. }), spec(), '/runtime/expired-termination')
  1197. await flush()
  1198. await fake.completeOutput()
  1199. handle.terminate()
  1200. await expect(handle.waitForExit()).resolves.toBe(true)
  1201. fake.alive = false
  1202. fake.handle.succeed(0)
  1203. await handle.done
  1204. })
  1205. it('makes batch stdin close failures best-effort', async () => {
  1206. const fake = new FakeSandbox()
  1207. vi.spyOn(fake.handle, 'sendStdin').mockRejectedValueOnce(new Error('closed'))
  1208. const handle = testHandle(runtime(fake), spec({
  1209. stdio: { stdin: { data: 'ignored' }, stdout: { maxBytes: 4 }, stderr: { maxBytes: 4 } },
  1210. }), '/runtime/stdin-closed')
  1211. await flush()
  1212. fake.finish()
  1213. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  1214. })
  1215. it('rejects malformed SDK process ids and non-command settlement failures', async () => {
  1216. const invalidPid = new FakeSandbox()
  1217. invalidPid.handle.pid = 0
  1218. const invalid = testHandle(runtime(invalidPid), spec(), '/runtime/invalid-pid')
  1219. await expect(invalid.done).rejects.toThrow(/invalid command pid 0/)
  1220. expect(invalidPid.handle.kills).toBe(1)
  1221. expect(invalidPid.removed).toContain('/runtime/invalid-pid/environment')
  1222. await expect(invalid.waitForExit()).resolves.toBe(true)
  1223. const failedRollback = new FakeSandbox()
  1224. failedRollback.handle.pid = 0
  1225. failedRollback.handle.killError = new Error('invalid handle kill failed')
  1226. const retained = testHandle(runtime(failedRollback), spec(), '/runtime/invalid-pid-retained')
  1227. await expect(retained.done).rejects.toThrow('invalid command pid rollback did not reach quiescence')
  1228. await expect(retained.waitForExit()).rejects.toThrow('invalid handle kill failed')
  1229. failedRollback.handle.killError = undefined
  1230. retained.terminate()
  1231. await expect(retained.waitForExit()).resolves.toBe(true)
  1232. const crashedFake = new FakeSandbox()
  1233. const crashed = testHandle(runtime(crashedFake), spec(), '/runtime/crashed')
  1234. await flush()
  1235. crashedFake.alive = false
  1236. crashedFake.handle.crash(new Error('command transport failed'))
  1237. await expect(crashed.done).rejects.toThrow('command transport failed')
  1238. })
  1239. it('rejects invalid or absent process-group publication', async () => {
  1240. const invalidGroup = new FakeSandbox()
  1241. invalidGroup.processGroupId = 'not-a-pid\n'
  1242. invalidGroup.delaysKill = true
  1243. invalidGroup.sdkKillStops = false
  1244. invalidGroup.afterProbe = () => { invalidGroup.alive = false }
  1245. const invalid = testHandle(runtime(invalidGroup), spec(), '/runtime/invalid-group')
  1246. await expect(invalid.done).rejects.toThrow(/invalid process-group id/)
  1247. expect(invalidGroup.handle.kills).toBe(1)
  1248. expect(invalidGroup.commandsSeen).toContain('kill -KILL -- -4242')
  1249. await expect(invalid.waitForExit()).resolves.toBe(true)
  1250. // A rewritten pid file must not aim the kill at every process (`-- -1`).
  1251. const unsafeGroup = new FakeSandbox()
  1252. unsafeGroup.processGroupId = '1\n'
  1253. unsafeGroup.delaysKill = true
  1254. unsafeGroup.sdkKillStops = false
  1255. unsafeGroup.afterProbe = () => { unsafeGroup.alive = false }
  1256. const unsafe = testHandle(runtime(unsafeGroup), spec(), '/runtime/unsafe-group')
  1257. await expect(unsafe.done).rejects.toThrow(/unsafe published process-group id 1/)
  1258. expect(unsafeGroup.commandsSeen).not.toContain('kill -KILL -- -1')
  1259. await expect(unsafe.waitForExit()).resolves.toBe(true)
  1260. const absentGroup = new FakeSandbox()
  1261. absentGroup.processGroupId = ''
  1262. const absent = testHandle(runtime(absentGroup), spec(), '/runtime/absent-group')
  1263. await flush()
  1264. absentGroup.finish()
  1265. await expect(absent.done).rejects.toThrow(/exited before publishing/)
  1266. expect(absentGroup.handle.kills).toBe(1)
  1267. expect(absentGroup.commandsSeen).toContain('kill -KILL -- -4242')
  1268. await expect(absent.waitForExit()).resolves.toBe(true)
  1269. })
  1270. it('keeps polling while a running command has not published its process group yet', async () => {
  1271. const fake = new FakeSandbox()
  1272. fake.processGroupReads.push('', '4242\n')
  1273. const handle = testHandle(runtime(fake), spec(), '/runtime/delayed-group-publication', 1)
  1274. await vi.waitFor(() => { expect(fake.processGroupReads).toEqual([]) })
  1275. fake.finish()
  1276. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  1277. await expect(handle.waitForExit()).resolves.toBe(true)
  1278. })
  1279. it('preserves publication failure and reports cleanup that cannot be verified', async () => {
  1280. const fake = new FakeSandbox()
  1281. fake.processGroupId = 'not-a-pid\n'
  1282. fake.signalError = new Error('rollback signal failed')
  1283. fake.handle.killError = new Error('SDK kill failed')
  1284. const handle = testHandle(runtime(fake), spec(), '/runtime/failed-rollback')
  1285. let failure: unknown
  1286. try {
  1287. await handle.done
  1288. } catch (error: unknown) {
  1289. failure = error
  1290. }
  1291. expect(failure).toBeInstanceOf(AggregateError)
  1292. if (!(failure instanceof AggregateError)) throw new Error('expected AggregateError')
  1293. expect(failure.message).toBe('subprocess-e2b: process-group publication failed and rollback did not reach quiescence')
  1294. const failures = Array.from(failure.errors as Iterable<unknown>)
  1295. expect(failures).toHaveLength(2)
  1296. expect(failures[0]).toBeInstanceOf(Error)
  1297. expect(failures[1]).toBeInstanceOf(Error)
  1298. if (!(failures[0] instanceof Error) || !(failures[1] instanceof Error)) throw new Error('expected nested errors')
  1299. expect(failures[0].message).toContain('invalid process-group id')
  1300. expect(failures[1].message).toContain('remained live after force termination')
  1301. expect(fake.handle.kills).toBe(1)
  1302. const bounded = new AbortController()
  1303. const waiting = handle.waitForExit(bounded.signal)
  1304. bounded.abort()
  1305. await expect(waiting).resolves.toBe(false)
  1306. handle.terminate()
  1307. await expect(handle.waitForExit()).resolves.toBe(true)
  1308. expect(fake.commandsSeen).toContain('kill -TERM -- -4242')
  1309. const naturallyGone = new FakeSandbox()
  1310. naturallyGone.processGroupId = 'not-a-pid\n'
  1311. naturallyGone.signalError = new Error('rollback signal failed')
  1312. naturallyGone.handle.killError = new Error('SDK kill failed')
  1313. const observed = testHandle(runtime(naturallyGone), spec(), '/runtime/failed-rollback-observed')
  1314. await expect(observed.done).rejects.toThrow('process-group publication failed')
  1315. naturallyGone.alive = false
  1316. await expect(observed.waitForExit()).resolves.toBe(true)
  1317. })
  1318. it('handles output backpressure and contains a stderr sink failure', async () => {
  1319. const fake = new FakeSandbox()
  1320. const handle = testHandle(runtime(fake), spec({
  1321. stdio: { stdin: 'ignore', stdout: 'pipe', stderr: 'pipe' },
  1322. }), '/runtime/backpressure')
  1323. await flush()
  1324. handle.stdout!.on('error', () => {})
  1325. const stdoutWrite = vi.spyOn(handle.stdout!, 'write').mockReturnValueOnce(false)
  1326. const stdoutPending = fake.stdout('blocked')
  1327. queueMicrotask(() => { handle.stdout!.emit('drain') })
  1328. await stdoutPending
  1329. stdoutWrite.mockRestore()
  1330. handle.stderr!.on('error', () => {})
  1331. const stderrWrite = vi.spyOn(handle.stderr!, 'write').mockReturnValueOnce(false)
  1332. const stderrPending = fake.stderr('broken')
  1333. queueMicrotask(() => { handle.stderr!.emit('error', new Error('sink failed')) })
  1334. await stderrPending
  1335. stderrWrite.mockRestore()
  1336. fake.finish()
  1337. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  1338. })
  1339. it('settles output backpressure when the consumer closes the pipe', async () => {
  1340. const fake = new FakeSandbox()
  1341. const handle = testHandle(runtime(fake), spec({
  1342. stdio: { stdin: 'ignore', stdout: 'pipe', stderr: { maxBytes: 4 } },
  1343. }), '/runtime/backpressure-close')
  1344. await flush()
  1345. const stdoutWrite = vi.spyOn(handle.stdout!, 'write').mockReturnValueOnce(false)
  1346. const pending = fake.stdout('discarded')
  1347. queueMicrotask(() => { handle.stdout!.destroy() })
  1348. await pending
  1349. stdoutWrite.mockRestore()
  1350. fake.finish()
  1351. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  1352. })
  1353. it('breaks output backpressure when termination owns the command', async () => {
  1354. const fake = new FakeSandbox()
  1355. const handle = testHandle(runtime(fake), spec({
  1356. stdio: { stdin: 'ignore', stdout: 'pipe', stderr: { maxBytes: 4 } },
  1357. }), '/runtime/backpressure-termination')
  1358. await flush()
  1359. const stdoutWrite = vi.spyOn(handle.stdout!, 'write').mockReturnValueOnce(false)
  1360. let released = false
  1361. const pending = fake.stdout('blocked').then(() => { released = true })
  1362. await Promise.resolve()
  1363. handle.terminate()
  1364. await flush()
  1365. const releasedByTermination = released
  1366. if (!released) handle.stdout!.emit('drain')
  1367. await pending
  1368. stdoutWrite.mockRestore()
  1369. await handle.done
  1370. expect(releasedByTermination).toBe(true)
  1371. })
  1372. it('settles backpressure when a synchronous pipe write starts termination', async () => {
  1373. const fake = new FakeSandbox()
  1374. const handle = testHandle(runtime(fake), spec({
  1375. stdio: { stdin: 'ignore', stdout: 'pipe', stderr: { maxBytes: 4 } },
  1376. }), '/runtime/backpressure-synchronous-termination')
  1377. await flush()
  1378. const stdoutWrite = vi.spyOn(handle.stdout!, 'write').mockImplementationOnce(() => {
  1379. handle.terminate()
  1380. return false
  1381. })
  1382. await expect(fake.stdout('blocked')).resolves.toBeUndefined()
  1383. stdoutWrite.mockRestore()
  1384. await handle.done
  1385. })
  1386. it('contains a pipe callback failure instead of rejecting command settlement', async () => {
  1387. const fake = new FakeSandbox()
  1388. const handle = testHandle(runtime(fake), spec({
  1389. stdio: { stdin: 'ignore', stdout: 'pipe', stderr: { maxBytes: 4 } },
  1390. }), '/runtime/pipe-error')
  1391. await flush()
  1392. const emitted = once(handle.stdout!, 'error')
  1393. handle.stdout!.destroy(new Error('consumer failed'))
  1394. await emitted
  1395. await fake.stdout('late output')
  1396. fake.finish()
  1397. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  1398. })
  1399. it('contains an already-gone group signal and escalates after a TERM transport failure', async () => {
  1400. const gone = new FakeSandbox()
  1401. gone.trapsTerm = true
  1402. gone.signalError = commandError(1)
  1403. const goneHandle = testHandle(runtime(gone), spec({ graceMs: 1 }), '/runtime/gone-signal')
  1404. await flush()
  1405. goneHandle.terminate()
  1406. await expect(goneHandle.done).resolves.toEqual({ exitCode: null, signal: 'SIGKILL' })
  1407. const failed = new FakeSandbox()
  1408. failed.signalError = new Error('signal transport failed')
  1409. const failedHandle = testHandle(runtime(failed), spec(), '/runtime/failed-signal')
  1410. await flush()
  1411. failedHandle.terminate()
  1412. await expect(failedHandle.done).resolves.toEqual({ exitCode: null, signal: 'SIGKILL' })
  1413. expect(failed.commandsSeen).toContain('kill -KILL -- -4242')
  1414. })
  1415. it('allows termination retry after both force transports fail', async () => {
  1416. const fake = new FakeSandbox()
  1417. fake.signalErrors.push(new Error('TERM transport failed'), new Error('KILL transport failed'))
  1418. fake.handle.killError = new Error('SDK kill failed')
  1419. const handle = testHandle(runtime(fake), spec({ graceMs: 1 }), '/runtime/retry-signal')
  1420. await flush()
  1421. handle.terminate()
  1422. await expect(handle.waitForExit()).rejects.toThrow('remained live after force termination')
  1423. fake.handle.killError = undefined
  1424. handle.terminate()
  1425. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGTERM' })
  1426. expect(fake.commandsSeen.filter(command => command.startsWith('kill -TERM '))).toHaveLength(2)
  1427. const missingGroup = new FakeSandbox()
  1428. missingGroup.trapsTerm = true
  1429. missingGroup.signalErrors.push(undefined, commandError(1))
  1430. missingGroup.handle.killError = new Error('SDK kill failed after group exit race')
  1431. const raced = testHandle(runtime(missingGroup), spec({ graceMs: 1 }), '/runtime/group-exit-race')
  1432. await flush()
  1433. raced.terminate()
  1434. await expect(raced.waitForExit()).rejects.toThrow('remained live after force termination')
  1435. missingGroup.handle.killError = undefined
  1436. raced.terminate()
  1437. await expect(raced.waitForExit()).resolves.toBe(true)
  1438. })
  1439. it('rejects an optimistic SDK kill while descendants survive a failed group KILL', async () => {
  1440. const fake = new FakeSandbox()
  1441. fake.trapsTerm = true
  1442. fake.sdkKillStops = false
  1443. fake.signalErrors.push(undefined, new Error('KILL transport failed'))
  1444. const handle = testHandle(runtime(fake), spec({ graceMs: 1 }), '/runtime/optimistic-sdk-kill')
  1445. await flush()
  1446. handle.terminate()
  1447. await expect(handle.waitForExit()).rejects.toThrow('remained live after force termination')
  1448. expect(fake.alive).toBe(true)
  1449. fake.sdkKillStops = true
  1450. handle.terminate()
  1451. await expect(handle.waitForExit()).resolves.toBe(true)
  1452. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGKILL' })
  1453. })
  1454. })
  1455. describe('E2BSubprocessRuntime', () => {
  1456. async function service(
  1457. fake = new FakeSandbox(),
  1458. providedRuntime: E2BRuntime = runtime(fake),
  1459. ): Promise<{ ctx: Context; fiber: Awaited<ReturnType<Context['plugin']>> }> {
  1460. const ctx = new Context()
  1461. ctx.provide('e2b', providedRuntime)
  1462. const fiber = await ctx.plugin(E2BSubprocessRuntime)
  1463. return { ctx, fiber }
  1464. }
  1465. it('registers handles and disposal terminates and joins live remote groups regardless of sandbox policy', async () => {
  1466. const fake = new FakeSandbox()
  1467. fake.trapsTerm = true
  1468. const { ctx, fiber } = await service(fake)
  1469. const handle = ctx.subprocess.spawn(spec({ graceMs: 1 }))
  1470. await flush()
  1471. await fiber.dispose()
  1472. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGKILL' })
  1473. expect(fake.alive).toBe(false)
  1474. })
  1475. it('awaits SDK settlement after the remote process group becomes quiescent', async () => {
  1476. const fake = new FakeSandbox()
  1477. fake.trapsTerm = true
  1478. const { ctx, fiber } = await service(fake)
  1479. const handle = ctx.subprocess.spawn(spec())
  1480. await flush()
  1481. fake.alive = false
  1482. let disposed = false
  1483. const disposing = fiber.dispose().then(() => { disposed = true })
  1484. await flush()
  1485. expect(disposed).toBe(false)
  1486. fake.finish()
  1487. await disposing
  1488. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  1489. })
  1490. it('reports a failed termination transaction from disposal instead of waiting on done', async () => {
  1491. const fake = new FakeSandbox()
  1492. fake.signalErrors.push(new Error('TERM transport failed'), new Error('KILL transport failed'))
  1493. fake.handle.killError = new Error('SDK kill failed')
  1494. const { ctx, fiber } = await service(fake)
  1495. const handle = ctx.subprocess.spawn(spec({ graceMs: 1 }))
  1496. await flush()
  1497. await expect(fiber.dispose()).resolves.toBeUndefined()
  1498. await expect(handle.waitForExit()).rejects.toThrow('remained live after force termination')
  1499. fake.handle.killError = undefined
  1500. handle.terminate()
  1501. await expect(handle.waitForExit()).resolves.toBe(true)
  1502. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGTERM' })
  1503. })
  1504. it('aggregates sibling cleanup failures instead of reporting only the first', async () => {
  1505. const { ctx, fiber } = await service()
  1506. const disposalErrors: unknown[] = []
  1507. ctx.logger.error = ((error: unknown) => { disposalErrors.push(error) }) as typeof ctx.logger.error
  1508. const first = {
  1509. terminate: vi.fn(),
  1510. waitForExit: vi.fn(async () => { throw new Error('first cleanup failed') }),
  1511. done: Promise.resolve({ exitCode: 0, signal: null }),
  1512. } as unknown as E2BSubprocessHandle
  1513. const second = {
  1514. terminate: vi.fn(),
  1515. waitForExit: vi.fn(async () => { throw new Error('second cleanup failed') }),
  1516. done: Promise.resolve({ exitCode: 0, signal: null }),
  1517. } as unknown as E2BSubprocessHandle
  1518. const live = (ctx.subprocess as unknown as { live: Set<E2BSubprocessHandle> }).live
  1519. live.add(first)
  1520. live.add(second)
  1521. await fiber.dispose()
  1522. const failure = disposalErrors[0]
  1523. expect(failure).toBeInstanceOf(AggregateError)
  1524. if (!(failure instanceof AggregateError)) throw new Error('expected AggregateError')
  1525. expect(failure.errors.map(error => (error as Error).message).sort()).toEqual([
  1526. 'first cleanup failed',
  1527. 'second cleanup failed',
  1528. ])
  1529. })
  1530. it('waits for every owned cleanup before reporting a disposal failure', async () => {
  1531. const { ctx, fiber } = await service()
  1532. const failed = {
  1533. terminate: vi.fn(),
  1534. waitForExit: vi.fn(async () => { throw new Error('cleanup failed') }),
  1535. done: Promise.resolve({ exitCode: 0, signal: null }),
  1536. } as unknown as E2BSubprocessHandle
  1537. let finishCleanup!: () => void
  1538. const cleanup = new Promise<boolean>((resolve) => {
  1539. finishCleanup = () => { resolve(true) }
  1540. })
  1541. const draining = {
  1542. terminate: vi.fn(),
  1543. waitForExit: vi.fn(() => cleanup),
  1544. done: Promise.resolve({ exitCode: 0, signal: null }),
  1545. } as unknown as E2BSubprocessHandle
  1546. const live = (ctx.subprocess as unknown as { live: Set<E2BSubprocessHandle> }).live
  1547. live.add(failed)
  1548. live.add(draining)
  1549. let disposed = false
  1550. const disposing = fiber.dispose().then(() => { disposed = true })
  1551. await flush()
  1552. expect(disposed).toBe(false)
  1553. finishCleanup()
  1554. await disposing
  1555. expect(live).toEqual(new Set([failed]))
  1556. })
  1557. it('releases naturally settled handles before later service disposal', async () => {
  1558. const fake = new FakeSandbox()
  1559. const { ctx, fiber } = await service(fake)
  1560. const handle = ctx.subprocess.spawn(spec())
  1561. await flush()
  1562. fake.finish()
  1563. await handle.done
  1564. await flush()
  1565. const signalsBefore = fake.commandsSeen.filter(command => command.startsWith('kill -')).length
  1566. await fiber.dispose()
  1567. expect(fake.commandsSeen.filter(command => command.startsWith('kill -')).length).toBe(signalsBefore)
  1568. })
  1569. it('contains a release liveness failure and retries quiescence during disposal', async () => {
  1570. const fake = new FakeSandbox()
  1571. let calls = 0
  1572. const reconnecting = runtime(fake, async () => {
  1573. calls += 1
  1574. if (calls === 2) throw new Error('transient liveness failure')
  1575. return fake.sandbox
  1576. })
  1577. const { ctx, fiber } = await service(fake, reconnecting)
  1578. const handle = ctx.subprocess.spawn(spec())
  1579. await flush()
  1580. fake.finish()
  1581. await handle.done
  1582. await flush()
  1583. await fiber.dispose()
  1584. expect(calls).toBeGreaterThanOrEqual(3)
  1585. })
  1586. it('contains spawn rejection while disposal is joining the pending handle', async () => {
  1587. const fake = new FakeSandbox()
  1588. fake.deferStart()
  1589. fake.backgroundError = new Error('start failed during disposal')
  1590. const { ctx, fiber } = await service(fake)
  1591. const subprocess = ctx.subprocess
  1592. const handle = subprocess.spawn(spec())
  1593. await vi.waitFor(() => { expect(fake.startOptions).toBeDefined() })
  1594. const disposing = fiber.dispose()
  1595. await flush()
  1596. expect(() => subprocess.spawn(spec())).toThrow('service is disposing')
  1597. fake.releaseStart()
  1598. await expect(disposing).resolves.toBeUndefined()
  1599. await expect(handle.done).rejects.toThrow('start failed during disposal')
  1600. })
  1601. it('validates synchronous spawn preconditions before handle or remote work', async () => {
  1602. const fake = new FakeSandbox()
  1603. const getSandbox = vi.fn(async () => fake.sandbox)
  1604. const { ctx } = await service(fake, runtime(fake, getSandbox))
  1605. const live = (ctx.subprocess as unknown as { live: Set<E2BSubprocessHandle> }).live
  1606. expect(() => ctx.subprocess.spawn(spec({ argv: [] }))).toThrow(/non-empty program/)
  1607. expect(() => ctx.subprocess.spawn(spec({ signal: AbortSignal.abort('stop') })))
  1608. .toThrow(new Error('aborted before spawn: stop'))
  1609. expect(() => ctx.subprocess.spawn(spec({ signal: AbortSignal.abort(null) })))
  1610. .toThrow(new Error('aborted before spawn: aborted'))
  1611. const throwingReason = { toString: () => { throw new Error('caller reason escaped') } }
  1612. expect(() => ctx.subprocess.spawn(spec({ signal: AbortSignal.abort(throwingReason) })))
  1613. .toThrow(new Error('aborted before spawn: aborted'))
  1614. for (const invalid of [
  1615. spec({ argv: ['bash\0'] }),
  1616. spec({ argv: ['bash', 'bad\0arg'] }),
  1617. spec({ cwd: 'bad\0cwd' }),
  1618. spec({ env: { REMOVED: undefined, 'BAD\0KEY': 'value' } }),
  1619. spec({ env: { BAD: 'bad\0value' } }),
  1620. ]) {
  1621. let thrown: unknown
  1622. try {
  1623. ctx.subprocess.spawn(invalid)
  1624. } catch (error) {
  1625. thrown = error
  1626. }
  1627. expect(thrown).toMatchObject({ name: 'TypeError', code: 'ERR_INVALID_ARG_VALUE' })
  1628. }
  1629. expect(getSandbox).not.toHaveBeenCalled()
  1630. expect(live).toEqual(new Set())
  1631. expect(fake.directories).toEqual([])
  1632. })
  1633. })