subprocess.spec.ts 71 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792
  1. import { once } from 'node:events'
  2. import { Context } from '@deepseek-ai/cordis'
  3. import {
  4. CommandExitError,
  5. FileNotFoundError,
  6. SandboxNotFoundError,
  7. type CommandHandle,
  8. type CommandResult,
  9. type Sandbox,
  10. } from '@deepseek-ai/dsh-e2b'
  11. import type E2BRuntime from '@deepseek-ai/dsh-e2b'
  12. import type { SubprocessSpawnSpec } from '@deepseek-ai/dsh-subprocess'
  13. import E2BSubprocessRuntime from '@deepseek-ai/dsh-subprocess-e2b'
  14. import * as E2BSubprocessInvariant from '../src/invariant.ts'
  15. import { E2BBase64Decoder, E2B_OUTPUT_COMPLETE_FRAME, E2BOutputReader } from '../src/output.ts'
  16. import { E2BSubprocessHandle } from '../src/process.ts'
  17. import InvariantRegistry from '@deepseek-ai/dsh-invariants'
  18. import { describe, expect, it, vi } from 'vitest'
  19. function commandError(exitCode: number): CommandExitError {
  20. return new CommandExitError({ exitCode, stdout: '', stderr: '', error: `exit ${exitCode}` })
  21. }
  22. interface StartOptions {
  23. background: true
  24. cwd: string
  25. stdin: boolean
  26. timeoutMs: number
  27. signal?: AbortSignal
  28. envs?: Record<string, string>
  29. onStdout?: (data: string) => void | Promise<void>
  30. onStderr?: (data: string) => void | Promise<void>
  31. }
  32. class FakeCommandHandle {
  33. pid = 4242
  34. readonly sent: Array<string | Uint8Array> = []
  35. closes = 0
  36. kills = 0
  37. disconnects = 0
  38. killError: unknown
  39. killResult = true
  40. disconnectError: unknown
  41. private readonly result = Promise.withResolvers<CommandResult>()
  42. private settled = false
  43. constructor(private readonly onKill: () => void = () => {}) {}
  44. wait(): Promise<CommandResult> {
  45. return this.result.promise
  46. }
  47. async sendStdin(data: string | Uint8Array): Promise<void> {
  48. this.sent.push(data)
  49. }
  50. async closeStdin(): Promise<void> {
  51. this.closes += 1
  52. }
  53. async kill(): Promise<boolean> {
  54. this.kills += 1
  55. if (this.killError !== undefined) throw this.killError
  56. this.onKill()
  57. return this.killResult
  58. }
  59. async disconnect(): Promise<void> {
  60. this.disconnects += 1
  61. if (this.disconnectError !== undefined) throw this.disconnectError
  62. }
  63. succeed(exitCode = 0): void {
  64. if (this.settled) return
  65. this.settled = true
  66. this.result.resolve({ exitCode, stdout: '', stderr: '' })
  67. }
  68. fail(exitCode: number): void {
  69. if (this.settled) return
  70. this.settled = true
  71. this.result.reject(commandError(exitCode))
  72. }
  73. crash(error: unknown): void {
  74. if (this.settled) return
  75. this.settled = true
  76. this.result.reject(error)
  77. }
  78. }
  79. class FakeSandbox {
  80. readonly handle: FakeCommandHandle
  81. readonly commandsSeen: string[] = []
  82. readonly writtenFiles: string[][] = []
  83. readonly writtenFileData = new Map<string, string>()
  84. readonly removed: string[] = []
  85. readonly directories: string[] = []
  86. startOptions: StartOptions | undefined
  87. backgroundError: unknown
  88. envError: unknown
  89. statusError: unknown
  90. nextRemoveError: unknown
  91. probeError: unknown
  92. signalError: unknown
  93. readonly signalErrors: unknown[] = []
  94. trapsTerm = false
  95. delaysKill = false
  96. delaysKillCompletion = false
  97. sdkKillStops = true
  98. alive = true
  99. zombieOnly = false
  100. ambient = 'PATH=/ambient/bin\0KEEP=safe\0UNICODE=你好\0NPM_TOKEN=secret\0DSH_STALE=old\0BROKEN\0=bad\0'
  101. environmentHome = '/home/user'
  102. environmentWire: string | undefined
  103. environmentRequest: ((signal: AbortSignal | undefined) => Promise<void>) | undefined
  104. processGroupId = '4242\n'
  105. exitStatus = ''
  106. statusReads = 0
  107. readonly processGroupReads: string[] = []
  108. afterStatusRead: (() => void) | undefined
  109. beforeProbe: (() => void) | undefined
  110. afterProbe: (() => void) | undefined
  111. private startGate: Promise<void> | undefined
  112. private openStart: (() => void) | undefined
  113. private processGroupReadGate: Promise<void> | undefined
  114. private openProcessGroupRead: (() => void) | undefined
  115. private signalGate: Promise<void> | undefined
  116. private openSignal: (() => void) | undefined
  117. constructor() {
  118. this.handle = new FakeCommandHandle(() => {
  119. if (this.sdkKillStops) {
  120. this.alive = false
  121. this.handle.fail(137)
  122. }
  123. })
  124. }
  125. deferStart(): void {
  126. const gate = Promise.withResolvers<undefined>()
  127. this.startGate = gate.promise
  128. this.openStart = () => { gate.resolve(undefined) }
  129. }
  130. releaseStart(): void {
  131. this.openStart?.()
  132. }
  133. deferProcessGroupRead(): void {
  134. const gate = Promise.withResolvers<undefined>()
  135. this.processGroupReadGate = gate.promise
  136. this.openProcessGroupRead = () => { gate.resolve(undefined) }
  137. }
  138. releaseProcessGroupRead(): void {
  139. this.openProcessGroupRead?.()
  140. }
  141. deferSignals(): void {
  142. const gate = Promise.withResolvers<undefined>()
  143. this.signalGate = gate.promise
  144. this.openSignal = () => { gate.resolve(undefined) }
  145. }
  146. releaseSignals(): void {
  147. this.openSignal?.()
  148. }
  149. finish(exitCode = 0): void {
  150. this.alive = false
  151. void this.completeOutput().then(
  152. () => {
  153. if (exitCode === 0) this.handle.succeed(0)
  154. else this.handle.fail(exitCode)
  155. },
  156. (error: unknown) => { this.handle.crash(error) },
  157. )
  158. }
  159. async completeOutput(): Promise<void> {
  160. await Promise.all([
  161. this.stdoutWire(`${E2B_OUTPUT_COMPLETE_FRAME}\n`),
  162. this.stderrWire(`${E2B_OUTPUT_COMPLETE_FRAME}\n`),
  163. ])
  164. }
  165. async stdout(data: string): Promise<void> {
  166. await this.stdoutWire(data.length === 0 ? '' : `${Buffer.from(data).toString('base64')}\n`)
  167. }
  168. async stderr(data: string): Promise<void> {
  169. await this.stderrWire(data.length === 0 ? '' : `${Buffer.from(data).toString('base64')}\n`)
  170. }
  171. async stdoutWire(data: string): Promise<void> {
  172. await this.startOptions?.onStdout?.(data)
  173. }
  174. async stderrWire(data: string): Promise<void> {
  175. await this.startOptions?.onStderr?.(data)
  176. }
  177. readonly sandbox = {
  178. sandboxId: 'fake',
  179. files: {
  180. makeDir: async (path: string): Promise<boolean> => {
  181. this.directories.push(path)
  182. return true
  183. },
  184. write: async (files: Array<{ path: string; data: string }>): Promise<object[]> => {
  185. this.writtenFiles.push(files.map(file => file.path))
  186. for (const file of files) this.writtenFileData.set(file.path, file.data)
  187. return files.map(() => ({}))
  188. },
  189. read: async (path: string): Promise<string> => {
  190. if (!path.endsWith('/exit-code')) {
  191. await this.processGroupReadGate
  192. return this.processGroupReads.shift() ?? this.processGroupId
  193. }
  194. if (this.statusError !== undefined) {
  195. const error = this.statusError
  196. this.statusError = undefined
  197. throw error
  198. }
  199. this.statusReads += 1
  200. this.afterStatusRead?.()
  201. return this.exitStatus
  202. },
  203. remove: async (path: string): Promise<void> => {
  204. this.removed.push(path)
  205. if (this.nextRemoveError !== undefined) {
  206. const error = this.nextRemoveError
  207. this.nextRemoveError = undefined
  208. throw error
  209. }
  210. },
  211. },
  212. commands: {
  213. run: async (command: string, options?: StartOptions | { signal?: AbortSignal }): Promise<CommandHandle | CommandResult> => {
  214. this.commandsSeen.push(command)
  215. if (command.includes('env -0 | base64')) {
  216. await this.environmentRequest?.(options?.signal)
  217. if (this.envError !== undefined) throw this.envError
  218. return {
  219. exitCode: 0,
  220. stdout: this.environmentWire ?? [this.environmentHome, this.ambient]
  221. .map(value => Buffer.from(value).toString('base64'))
  222. .join('\n'),
  223. stderr: '',
  224. }
  225. }
  226. if (command.startsWith('set -o pipefail; ps -eo pgid=,stat=')) {
  227. this.beforeProbe?.()
  228. if (options?.signal?.aborted === true) throw new DOMException('aborted', 'AbortError')
  229. if (this.probeError !== undefined) {
  230. const error = this.probeError
  231. this.probeError = undefined
  232. throw error
  233. }
  234. const stdout = this.alive && !this.zombieOnly ? 'live\n' : ''
  235. this.afterProbe?.()
  236. return { exitCode: 0, stdout, stderr: '' }
  237. }
  238. if (command.startsWith('kill -TERM ')) {
  239. await this.signalGate
  240. const error = this.signalErrors.shift() ?? this.signalError
  241. if (error !== undefined) {
  242. if (this.signalErrors.length === 0) this.signalError = undefined
  243. throw error
  244. }
  245. if (!this.trapsTerm) {
  246. this.alive = false
  247. this.handle.fail(143)
  248. }
  249. return { exitCode: 0, stdout: '', stderr: '' }
  250. }
  251. if (command.startsWith('kill -KILL ')) {
  252. await this.signalGate
  253. const error = this.signalErrors.shift() ?? this.signalError
  254. if (error !== undefined) {
  255. if (this.signalErrors.length === 0) this.signalError = undefined
  256. throw error
  257. }
  258. if (!this.delaysKill) this.alive = false
  259. if (!this.delaysKillCompletion) this.handle.fail(137)
  260. return { exitCode: 0, stdout: '', stderr: '' }
  261. }
  262. if ((options as StartOptions | undefined)?.background === true) {
  263. this.startOptions = options as StartOptions
  264. await this.startGate
  265. if (this.backgroundError !== undefined) throw this.backgroundError
  266. return this.handle as unknown as CommandHandle
  267. }
  268. return { exitCode: 0, stdout: '', stderr: '' }
  269. },
  270. },
  271. } as unknown as Sandbox
  272. }
  273. function spec(overrides: Partial<SubprocessSpawnSpec> = {}): SubprocessSpawnSpec {
  274. return {
  275. argv: ['bash', '-c', 'printf ok'],
  276. cwd: '/workspace',
  277. stdio: {
  278. stdin: 'ignore',
  279. stdout: { maxBytes: 4, spill: { maxBytes: 16 } },
  280. stderr: { maxBytes: 4 },
  281. },
  282. graceMs: 5,
  283. ...overrides,
  284. }
  285. }
  286. function runtime(fake: FakeSandbox, getSandbox: () => Promise<Sandbox> = async () => fake.sandbox): E2BRuntime {
  287. return {
  288. cwd: '/workspace',
  289. runtimeRoot: '/workspace/.dsh-e2b',
  290. getSandbox,
  291. } as unknown as E2BRuntime
  292. }
  293. async function flush(): Promise<void> {
  294. await new Promise(resolve => setTimeout(resolve, 0))
  295. }
  296. /** Construct the handle under test with the config default the service would pass. */
  297. function testHandle(
  298. runtime: ConstructorParameters<typeof E2BSubprocessHandle>[0],
  299. spec: ConstructorParameters<typeof E2BSubprocessHandle>[1],
  300. stateDir: string,
  301. pollMs = 20,
  302. ): E2BSubprocessHandle {
  303. return new E2BSubprocessHandle(runtime, spec, stateDir, pollMs)
  304. }
  305. describe('E2BOutputReader', () => {
  306. it('decodes base64 across arbitrary callback boundaries and rejects malformed framing', () => {
  307. const decoder = new E2BBase64Decoder()
  308. expect(decoder.push('')).toEqual(Buffer.alloc(0))
  309. expect(decoder.push('5')).toEqual(Buffer.alloc(0))
  310. expect(decoder.push('L2')).toEqual(Buffer.alloc(0))
  311. expect(decoder.push('g\n').toString()).toBe('你')
  312. expect(decoder.push('YQ==\nYg==\n').toString()).toBe('ab')
  313. expect(decoder.push(`${Buffer.from([0, 255]).toString('base64')}\n`)).toEqual(Buffer.from([0, 255]))
  314. expect(decoder.push(`${E2B_OUTPUT_COMPLETE_FRAME}\n`)).toEqual(Buffer.alloc(0))
  315. decoder.finish()
  316. expect(() => new E2BBase64Decoder().push('%\n')).toThrow('invalid base64')
  317. expect(() => new E2BBase64Decoder().push('AB==\n')).toThrow('invalid base64')
  318. expect(() => decoder.push(`${E2B_OUTPUT_COMPLETE_FRAME}\n`)).toThrow('duplicate output transport completion')
  319. expect(() => decoder.push('YQ==\n')).toThrow('continued after completion')
  320. const truncated = new E2BBase64Decoder()
  321. truncated.push('YQ')
  322. expect(() => { truncated.finish() }).toThrow('truncated base64')
  323. expect(() => { new E2BBase64Decoder().finish() }).toThrow('incomplete output transport')
  324. const interrupted = new E2BBase64Decoder()
  325. interrupted.push('YQ')
  326. expect(() => { interrupted.finish(false) }).not.toThrow()
  327. })
  328. it('keeps a byte-exact tail with independent whole-stream cursors', () => {
  329. const reader = new E2BOutputReader(4, 10, '/remote/spill')
  330. reader.push(Buffer.alloc(0))
  331. reader.push(Buffer.from('ab'))
  332. reader.push(Buffer.from('cdef'))
  333. expect(reader.size).toBe(6)
  334. expect(reader.readFrom(0)).toEqual({ text: 'cdef', nextOffset: 6, lossy: true, spillPath: '/remote/spill' })
  335. expect(reader.readFrom(2)).toEqual({ text: 'cdef', nextOffset: 6, lossy: false })
  336. expect(reader.readFrom(5)).toEqual({ text: 'f', nextOffset: 6, lossy: false })
  337. expect(reader.readFrom(99)).toEqual({ text: '', nextOffset: 6, lossy: false })
  338. reader.invalidateSpill()
  339. expect(reader.readFrom(0)).toEqual({ text: 'cdef', nextOffset: 6, lossy: true })
  340. })
  341. it('drops whole head chunks and withholds absent or over-cap spills', () => {
  342. const withoutSpill = new E2BOutputReader(2, undefined, '/unused')
  343. withoutSpill.push(Buffer.from('ab'))
  344. withoutSpill.push(Buffer.from('cd'))
  345. expect(withoutSpill.readFrom(0)).toEqual({ text: 'cd', nextOffset: 4, lossy: true })
  346. const overCap = new E2BOutputReader(2, 3, '/too-small')
  347. overCap.push(Buffer.from('abcd'))
  348. expect(overCap.readFrom(0)).toEqual({ text: 'cd', nextOffset: 4, lossy: true })
  349. })
  350. })
  351. describe('E2BSubprocessHandle', () => {
  352. it('starts asynchronously, keeps secrets out of the command, and supports deferred piped stdin/output', async () => {
  353. const fake = new FakeSandbox()
  354. fake.processGroupId = '4343\n'
  355. fake.deferStart()
  356. const handle = testHandle(runtime(fake), spec({
  357. argv: ['tool', 'argument with spaces'],
  358. stdio: { stdin: 'pipe', stdout: 'pipe', stderr: { maxBytes: 8, spill: { maxBytes: 32 } } },
  359. env: {
  360. PATH: '/bin',
  361. 'FOO-BAR': 'hyphen-value',
  362. '--split-string': 'literal-value',
  363. DEEPSEEK_API_KEY: 'explicit-secret',
  364. DSH_MODE: 'test',
  365. // The seam's tombstone: an explicit undefined removes the ambient entry.
  366. KEEP: undefined,
  367. },
  368. }), '/workspace/.dsh-e2b/processes/one')
  369. expect(handle.pid).toBe(-1)
  370. handle.stdin!.write('hello')
  371. handle.stdin!.end()
  372. fake.releaseStart()
  373. await flush()
  374. expect(handle.pid).toBe(4343)
  375. expect(fake.handle.sent.map(value => String(value))).toEqual(['hello'])
  376. expect(fake.handle.closes).toBe(1)
  377. const controlEnvs = fake.startOptions?.envs
  378. expect(controlEnvs?.HOME).toMatch(/^\/\.dsh-e2b-control-/)
  379. expect(controlEnvs).toEqual({
  380. TERM: 'dumb',
  381. NPM_TOKEN: '',
  382. DSH_STALE: '',
  383. HOME: controlEnvs?.HOME,
  384. })
  385. const command = fake.commandsSeen.find(value => value.includes('exec "$dsh_e2b_env_bin" -i'))!
  386. expect(command).toContain('"$dsh_e2b_setsid" --wait -- "$dsh_e2b_bash" -c')
  387. expect(command).not.toContain('DEEPSEEK_API_KEY')
  388. expect(command).not.toContain('DSH_MODE')
  389. expect(command).not.toContain('FOO-BAR')
  390. expect(command).not.toContain('explicit-secret')
  391. expect(command).not.toContain('hyphen-value')
  392. expect(command).not.toContain('${!dsh_e2b_name}')
  393. const environmentProbe = fake.commandsSeen.find(value => value.includes('env -0 | base64'))
  394. expect(environmentProbe).toContain('getent passwd "$(id -u)"')
  395. expect(environmentProbe).toContain('test -n "$dsh_e2b_home" -a -d "$dsh_e2b_home"')
  396. expect(environmentProbe).not.toContain('"$PWD"')
  397. expect(command).toContain('mapfile -d')
  398. expect(command).toContain('dsh_e2b_node="$(command -v node)"')
  399. expect(command).toContain('"$dsh_e2b_env_bin" -i "$dsh_e2b_node" -e')
  400. expect(command).toContain('"$dsh_e2b_env_bin" -i -- "${dsh_e2b_env[@]}" "$@"')
  401. expect(command).toContain('exec "$dsh_e2b_env_bin" -i -- "${dsh_e2b_env[@]}"')
  402. expect(command).toContain('>&2 2>/dev/null')
  403. expect(command).not.toContain('2>/dev/null >&2')
  404. expect(command).toContain('base64')
  405. expect(fake.writtenFiles[0]).toEqual([
  406. '/workspace/.dsh-e2b/processes/one/pid',
  407. '/workspace/.dsh-e2b/processes/one/exit-code',
  408. '/workspace/.dsh-e2b/processes/one/environment',
  409. '/workspace/.dsh-e2b/processes/one/stderr.log',
  410. ])
  411. expect(fake.writtenFileData.get('/workspace/.dsh-e2b/processes/one/environment')).toBe(
  412. 'PATH=/bin\0UNICODE=你好\0HOME=/home/user\0FOO-BAR=hyphen-value\0--split-string=literal-value\0DEEPSEEK_API_KEY=explicit-secret\0DSH_MODE=test\0',
  413. )
  414. let piped = ''
  415. handle.stdout!.on('data', (chunk) => { piped += String(chunk) })
  416. await fake.stdout('pipe-data')
  417. await fake.stderr('err')
  418. fake.finish()
  419. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  420. expect(piped).toBe('pipe-data')
  421. expect(handle.collected.stderr!.readFrom(0)).toMatchObject({ text: 'err', lossy: false })
  422. expect(fake.removed).toContain('/workspace/.dsh-e2b/processes/one/stderr.log')
  423. await expect(handle.waitForExit()).resolves.toBe(true)
  424. })
  425. it('rejects an unrepresentable graceMs before any remote work', () => {
  426. const ctx = new Context()
  427. const service = Object.create(E2BSubprocessRuntime.prototype) as E2BSubprocessRuntime
  428. Reflect.set(service, 'disposing', false)
  429. Reflect.set(service, 'ctx', ctx)
  430. for (const graceMs of [0, -1, Number.NaN, Number.POSITIVE_INFINITY]) {
  431. expect(() => service.spawn(spec({ graceMs }))).toThrow('graceMs must be a positive finite number')
  432. void expect(service.spawnTerminal({
  433. argv: ['bash'], cwd: '/w', rows: 24, cols: 80, graceMs,
  434. })).rejects.toThrow('graceMs must be a positive finite number')
  435. }
  436. })
  437. it('rejects malformed environment entries before command start', async () => {
  438. for (const env of [{ 'BAD=NAME': 'x' }, { BAD: 'x\0INJECTED=1' }]) {
  439. const fake = new FakeSandbox()
  440. const handle = testHandle(runtime(fake), spec({ env }), '/runtime/invalid-environment')
  441. await expect(handle.done).rejects.toThrow('environment entries')
  442. expect(fake.startOptions).toBeUndefined()
  443. expect(fake.removed).toContain('/runtime/invalid-environment')
  444. }
  445. })
  446. it('preserves UTF-8 bytes when the ASCII transport is split across callbacks', async () => {
  447. const fake = new FakeSandbox()
  448. const handle = testHandle(runtime(fake), spec({
  449. stdio: { stdin: 'ignore', stdout: 'pipe', stderr: { maxBytes: 4 } },
  450. }), '/runtime/split-utf8')
  451. await flush()
  452. const chunks: Buffer[] = []
  453. handle.stdout!.on('data', (chunk: Buffer) => { chunks.push(chunk) })
  454. for (const character of `${Buffer.from('A你好B').toString('base64')}\n`) {
  455. await fake.stdoutWire(character)
  456. }
  457. fake.finish()
  458. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  459. expect(Buffer.concat(chunks).toString('utf8')).toBe('A你好B')
  460. })
  461. it('rejects malformed output transport without confusing it with a consumer sink failure', async () => {
  462. const fake = new FakeSandbox()
  463. const handle = testHandle(runtime(fake), spec(), '/runtime/malformed-output')
  464. await flush()
  465. await fake.stdoutWire('%\n')
  466. fake.finish()
  467. await expect(handle.done).rejects.toThrow('invalid base64 output transport')
  468. const stderrFake = new FakeSandbox()
  469. const stderrHandle = testHandle(runtime(stderrFake), spec(), '/runtime/malformed-stderr')
  470. await flush()
  471. await stderrFake.stderrWire('%\n')
  472. stderrFake.finish()
  473. await expect(stderrHandle.done).rejects.toThrow('invalid base64 output transport')
  474. })
  475. it('rejects a naturally completed command whose encoder omits its completion frame', async () => {
  476. const fake = new FakeSandbox()
  477. const handle = testHandle(runtime(fake), spec(), '/runtime/incomplete-output')
  478. await flush()
  479. fake.alive = false
  480. fake.handle.succeed(0)
  481. await expect(handle.done).rejects.toThrow('incomplete output transport')
  482. })
  483. it('bounds descendant-held output draining and withholds the incomplete spill', async () => {
  484. const fake = new FakeSandbox()
  485. const handle = testHandle(runtime(fake), spec({ graceMs: 5 }), '/runtime/drain-bound')
  486. await flush()
  487. await fake.stdout('leader-output')
  488. fake.exitStatus = '0\n'
  489. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  490. expect(fake.handle.disconnects).toBe(1)
  491. expect(handle.collected.stdout?.readFrom(0)).toEqual({
  492. text: 'tput',
  493. nextOffset: 13,
  494. lossy: true,
  495. })
  496. expect(fake.removed).toContain('/runtime/drain-bound/stdout.log')
  497. handle.terminate()
  498. await expect(handle.waitForExit()).resolves.toBe(true)
  499. })
  500. it('releases an inherited-output callback blocked on host backpressure at drain expiry', async () => {
  501. const fake = new FakeSandbox()
  502. const written: string[] = []
  503. const stdoutWrite = vi.spyOn(process.stdout, 'write').mockImplementation(((chunk: Uint8Array) => {
  504. written.push(Buffer.from(chunk).toString())
  505. return false
  506. }) as typeof process.stdout.write)
  507. try {
  508. const handle = testHandle(runtime(fake), spec({
  509. graceMs: 5,
  510. stdio: { stdin: 'ignore', stdout: 'inherit', stderr: { maxBytes: 4 } },
  511. }), '/runtime/inherit-backpressure')
  512. await flush()
  513. let callbackSettled = false
  514. const blocked = fake.stdout('blocked bytes').then(() => { callbackSettled = true })
  515. await flush()
  516. expect(callbackSettled).toBe(false)
  517. fake.exitStatus = '0\n'
  518. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  519. await blocked
  520. expect(callbackSettled).toBe(true)
  521. expect(written.join('')).toBe('blocked bytes')
  522. expect(fake.handle.disconnects).toBe(1)
  523. handle.terminate()
  524. await expect(handle.waitForExit()).resolves.toBe(true)
  525. } finally {
  526. stdoutWrite.mockRestore()
  527. }
  528. })
  529. it('waits for lossless raw-pipe output after the direct status is published', async () => {
  530. const fake = new FakeSandbox()
  531. const handle = testHandle(runtime(fake), spec({
  532. graceMs: 1,
  533. stdio: { stdin: 'ignore', stdout: 'pipe', stderr: { maxBytes: 4 } },
  534. }), '/runtime/pipe-drain')
  535. let output = ''
  536. handle.stdout!.on('data', (chunk) => { output += String(chunk) })
  537. await flush()
  538. fake.exitStatus = '0\n'
  539. let settled = false
  540. void handle.done.then(() => { settled = true })
  541. await new Promise(resolve => setTimeout(resolve, 50))
  542. expect(settled).toBe(false)
  543. expect(fake.handle.disconnects).toBe(0)
  544. expect(fake.statusReads).toBe(0)
  545. await fake.stdout('complete protocol frame')
  546. fake.finish()
  547. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  548. expect(output).toBe('complete protocol frame')
  549. expect(fake.statusReads).toBe(1)
  550. })
  551. it('accepts clean encoder completion inside the output-drain grace', async () => {
  552. const fake = new FakeSandbox()
  553. const handle = testHandle(runtime(fake), spec({ graceMs: 100 }), '/runtime/drain-complete')
  554. await flush()
  555. fake.exitStatus = '0\n'
  556. fake.afterStatusRead = () => {
  557. fake.afterStatusRead = undefined
  558. setTimeout(() => { fake.finish() }, 0)
  559. }
  560. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  561. expect(fake.handle.disconnects).toBe(0)
  562. })
  563. it('preserves a published exit code when requested termination outlives output draining', async () => {
  564. const fake = new FakeSandbox()
  565. fake.trapsTerm = true
  566. fake.delaysKill = true
  567. fake.delaysKillCompletion = true
  568. fake.sdkKillStops = false
  569. const handle = testHandle(runtime(fake), spec({ graceMs: 5 }), '/runtime/drain-signal')
  570. await flush()
  571. handle.terminate()
  572. await vi.waitFor(() => { expect(fake.commandsSeen).toContain('kill -KILL -- -4242') })
  573. fake.exitStatus = '0\n'
  574. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  575. expect(fake.handle.disconnects).toBe(1)
  576. fake.alive = false
  577. handle.terminate()
  578. await expect(handle.waitForExit()).resolves.toBe(true)
  579. })
  580. it('preserves a published nonzero exit code when termination settles the SDK inside the drain grace', async () => {
  581. const fake = new FakeSandbox()
  582. const handle = testHandle(runtime(fake), spec({ graceMs: 100 }), '/runtime/drain-signal-settled')
  583. await flush()
  584. fake.exitStatus = '7\n'
  585. fake.afterStatusRead = () => {
  586. fake.afterStatusRead = undefined
  587. handle.terminate()
  588. }
  589. await expect(handle.done).resolves.toEqual({ exitCode: 7, signal: null })
  590. await expect(handle.waitForExit()).resolves.toBe(true)
  591. })
  592. it('rejects an invalid direct-command exit status', async () => {
  593. const fake = new FakeSandbox()
  594. const handle = testHandle(runtime(fake), spec(), '/runtime/invalid-status')
  595. await flush()
  596. fake.exitStatus = '999\n'
  597. await expect(handle.done).rejects.toThrow('invalid exit code')
  598. handle.terminate()
  599. await expect(handle.waitForExit()).resolves.toBe(true)
  600. })
  601. it('rolls back a published process group before rejecting a monitoring failure', async () => {
  602. const fake = new FakeSandbox()
  603. fake.statusError = new Error('status transport failed')
  604. const handle = testHandle(runtime(fake), spec(), '/runtime/status-failure')
  605. await expect(handle.done).rejects.toThrow('status transport failed')
  606. expect(fake.commandsSeen).toContain('kill -TERM -- -4242')
  607. expect(fake.alive).toBe(false)
  608. await expect(handle.waitForExit()).resolves.toBe(true)
  609. const failed = new FakeSandbox()
  610. failed.statusError = new Error('status transport failed')
  611. failed.signalErrors.push(new Error('TERM transport failed'), new Error('KILL transport failed'))
  612. failed.handle.killError = new Error('SDK kill failed')
  613. const retained = testHandle(runtime(failed), spec({ graceMs: 1 }), '/runtime/status-cleanup-failure')
  614. await expect(retained.done).rejects.toThrow(
  615. 'command monitoring failed and process-group rollback did not reach quiescence',
  616. )
  617. expect(failed.alive).toBe(true)
  618. failed.handle.killError = undefined
  619. retained.terminate()
  620. await expect(retained.waitForExit()).resolves.toBe(true)
  621. // A state-cleanup failure on top preserves the rollback failure instead of
  622. // re-aggregating only the original monitoring error.
  623. const triple = new FakeSandbox()
  624. triple.statusError = new Error('status transport failed')
  625. triple.signalErrors.push(new Error('TERM transport failed'), new Error('KILL transport failed'))
  626. triple.handle.killError = new Error('SDK kill failed')
  627. triple.nextRemoveError = new Error('state cleanup failed')
  628. const tripleHandle = testHandle(runtime(triple), spec({ graceMs: 1 }), '/runtime/triple-failure')
  629. const failure = await tripleHandle.done.catch((error: unknown) => error as AggregateError)
  630. expect(failure).toBeInstanceOf(AggregateError)
  631. expect((failure as AggregateError).message).toContain('private state cleanup failed')
  632. const nested = (failure as AggregateError).errors[0] as AggregateError
  633. expect(nested.message).toContain('rollback did not reach quiescence')
  634. triple.handle.killError = undefined
  635. tripleHandle.terminate()
  636. await expect(tripleHandle.waitForExit()).resolves.toBe(true)
  637. })
  638. it('surfaces deferred piped-stdin write and close failures as stream errors', async () => {
  639. const writeFake = new FakeSandbox()
  640. writeFake.deferStart()
  641. vi.spyOn(writeFake.handle, 'sendStdin').mockRejectedValueOnce('stdin rejected')
  642. const writeHandle = testHandle(runtime(writeFake), spec({
  643. stdio: { stdin: 'pipe', stdout: { maxBytes: 4 }, stderr: { maxBytes: 4 } },
  644. }), '/runtime/stdin-write-error')
  645. const writeError = once(writeHandle.stdin!, 'error')
  646. writeHandle.stdin!.write('input')
  647. writeFake.releaseStart()
  648. await expect(writeError).resolves.toMatchObject([{ message: 'stdin rejected' }])
  649. writeFake.finish()
  650. await writeHandle.done
  651. const closeFake = new FakeSandbox()
  652. vi.spyOn(closeFake.handle, 'closeStdin').mockRejectedValueOnce(new Error('close rejected'))
  653. const closeHandle = testHandle(runtime(closeFake), spec({
  654. stdio: { stdin: 'pipe', stdout: { maxBytes: 4 }, stderr: { maxBytes: 4 } },
  655. }), '/runtime/stdin-close-error')
  656. await flush()
  657. const closeError = once(closeHandle.stdin!, 'error')
  658. closeHandle.stdin!.end()
  659. await expect(closeError).resolves.toMatchObject([{ message: 'close rejected' }])
  660. closeFake.finish()
  661. await closeHandle.done
  662. })
  663. it('collects bounded tails, retains valid spills, and maps natural nonzero exits', async () => {
  664. const fake = new FakeSandbox()
  665. const handle = testHandle(runtime(fake), spec({
  666. stdio: {
  667. stdin: { data: 'batch' },
  668. stdout: { maxBytes: 4, spill: { maxBytes: 16 } },
  669. stderr: { maxBytes: 3 },
  670. },
  671. }), '/runtime/two')
  672. await flush()
  673. await fake.stdout('abcdef')
  674. await fake.stderr('12345')
  675. fake.finish(7)
  676. await expect(handle.done).resolves.toEqual({ exitCode: 7, signal: null })
  677. expect(fake.handle.sent).toEqual(['batch'])
  678. expect(fake.handle.closes).toBe(1)
  679. expect(handle.collected.stdout!.readFrom(0)).toEqual({
  680. text: 'cdef',
  681. nextOffset: 6,
  682. lossy: true,
  683. spillPath: '/runtime/two/stdout.log',
  684. })
  685. expect(handle.collected.stderr!.readFrom(0)).toEqual({ text: '345', nextOffset: 5, lossy: true })
  686. expect(fake.removed).not.toContain('/runtime/two/stdout.log')
  687. })
  688. it('removes a spill once the complete stream exceeds its cap', async () => {
  689. const fake = new FakeSandbox()
  690. const handle = testHandle(runtime(fake), spec({
  691. stdio: { stdin: 'ignore', stdout: { maxBytes: 2, spill: { maxBytes: 3 } }, stderr: 'inherit' },
  692. }), '/runtime/oversize')
  693. await flush()
  694. await fake.stdout('abcd')
  695. await fake.stderr('')
  696. fake.finish()
  697. await handle.done
  698. expect(handle.collected.stdout!.readFrom(0)).toEqual({ text: 'cd', nextOffset: 4, lossy: true })
  699. expect(fake.removed).toContain('/runtime/oversize/stdout.log')
  700. const command = fake.commandsSeen.find(value => value.includes('dsh_e2b_tee='))!
  701. expect(command).toContain('"$dsh_e2b_head" -c 3')
  702. expect(command).toContain('/runtime/oversize/stdout.log')
  703. expect(command).toContain('"$dsh_e2b_tee" --output-error=warn-nopipe')
  704. expect(command).not.toContain('tee -a')
  705. })
  706. it('contains remote spill-removal failures and routes empty inherited output', async () => {
  707. const fake = new FakeSandbox()
  708. fake.nextRemoveError = new Error('already removed')
  709. const handle = testHandle(runtime(fake), spec({
  710. stdio: { stdin: 'ignore', stdout: 'inherit', stderr: { maxBytes: 4, spill: { maxBytes: 8 } } },
  711. }), '/runtime/remove-error')
  712. await flush()
  713. await fake.stdout('')
  714. await fake.stderr('')
  715. fake.finish()
  716. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  717. expect(fake.removed).toContain('/runtime/remove-error/stderr.log')
  718. })
  719. it('terminates a process group with TERM and reports the signal outcome', async () => {
  720. const fake = new FakeSandbox()
  721. const handle = testHandle(runtime(fake), spec(), '/runtime/term')
  722. await flush()
  723. handle.terminate()
  724. handle.terminate()
  725. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGTERM' })
  726. await expect(handle.waitForExit()).resolves.toBe(true)
  727. expect(fake.commandsSeen).toContain('kill -TERM -- -4242')
  728. expect(fake.commandsSeen).not.toContain('kill -KILL -- -4242')
  729. const signals = fake.commandsSeen.filter(command => command.startsWith('kill -')).length
  730. fake.alive = true
  731. handle.terminate()
  732. await flush()
  733. expect(fake.alive).toBe(true)
  734. expect(fake.commandsSeen.filter(command => command.startsWith('kill -'))).toHaveLength(signals)
  735. })
  736. it('makes termination a permanent no-op after natural quiescence is observed', async () => {
  737. const fake = new FakeSandbox()
  738. const handle = testHandle(runtime(fake), spec(), '/runtime/natural-quiescence')
  739. await flush()
  740. fake.finish()
  741. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  742. await expect(handle.waitForExit()).resolves.toBe(true)
  743. const signals = fake.commandsSeen.filter(command => command.startsWith('kill -')).length
  744. fake.alive = true
  745. handle.terminate()
  746. await flush()
  747. expect(fake.alive).toBe(true)
  748. expect(fake.commandsSeen.filter(command => command.startsWith('kill -'))).toHaveLength(signals)
  749. })
  750. it('treats a zombie-only process group as quiescent', async () => {
  751. const fake = new FakeSandbox()
  752. fake.zombieOnly = true
  753. const handle = testHandle(runtime(fake), spec(), '/runtime/zombie-quiescence')
  754. await flush()
  755. await expect(handle.waitForExit()).resolves.toBe(true)
  756. expect(fake.commandsSeen).toContain(
  757. 'set -o pipefail; ps -eo pgid=,stat= | awk \'$1 == 4242 && $2 !~ /^[ZXx]/ { live=1 } END { if (live) print "live" }\'',
  758. )
  759. fake.finish()
  760. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  761. })
  762. it('keeps proven quiescence after a concurrent termination transport fails', async () => {
  763. const fake = new FakeSandbox()
  764. fake.signalErrors.push(new Error('TERM transport failed'), new Error('KILL transport failed'))
  765. fake.handle.killError = new Error('SDK kill failed')
  766. fake.deferSignals()
  767. const handle = testHandle(runtime(fake), spec(), '/runtime/quiescent-race')
  768. await flush()
  769. handle.terminate()
  770. await vi.waitFor(() => { expect(fake.commandsSeen).toContain('kill -TERM -- -4242') })
  771. fake.alive = false
  772. await expect(handle.waitForExit()).resolves.toBe(true)
  773. fake.probeError = new Error('post-quiescence probe failed')
  774. fake.releaseSignals()
  775. await vi.waitFor(() => { expect(fake.handle.kills).toBe(1) })
  776. fake.finish()
  777. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  778. const signals = fake.commandsSeen.filter(command => command.startsWith('kill -')).length
  779. handle.terminate()
  780. await expect(handle.waitForExit()).resolves.toBe(true)
  781. expect(fake.commandsSeen.filter(command => command.startsWith('kill -'))).toHaveLength(signals)
  782. })
  783. it('escalates a TERM-trapping process group to KILL and uses the SDK kill as fallback', async () => {
  784. const fake = new FakeSandbox()
  785. fake.trapsTerm = true
  786. fake.handle.killError = new Error('already gone')
  787. const handle = testHandle(runtime(fake), spec({ graceMs: 1 }), '/runtime/kill')
  788. await flush()
  789. handle.terminate()
  790. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGKILL' })
  791. await expect(handle.waitForExit()).resolves.toBe(true)
  792. expect(fake.commandsSeen).toContain('kill -KILL -- -4242')
  793. expect(fake.handle.kills).toBe(1)
  794. })
  795. it('keeps force cleanup retryable until quiescence is proven', async () => {
  796. const fake = new FakeSandbox()
  797. fake.trapsTerm = true
  798. fake.delaysKill = true
  799. fake.delaysKillCompletion = true
  800. fake.sdkKillStops = false
  801. const handle = testHandle(runtime(fake), spec({ graceMs: 1 }), '/runtime/termination-fence')
  802. await flush()
  803. handle.terminate()
  804. await vi.waitFor(() => { expect(fake.handle.kills).toBe(1) })
  805. await expect(handle.waitForExit()).rejects.toThrow('remained live after force termination')
  806. expect(fake.alive).toBe(true)
  807. fake.delaysKill = false
  808. fake.delaysKillCompletion = false
  809. fake.sdkKillStops = true
  810. handle.terminate()
  811. await expect(handle.waitForExit()).resolves.toBe(true)
  812. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGKILL' })
  813. })
  814. it('honors termination requested before asynchronous startup finishes', async () => {
  815. const fake = new FakeSandbox()
  816. fake.deferStart()
  817. const handle = testHandle(runtime(fake), spec(), '/runtime/deferred-kill')
  818. handle.terminate()
  819. fake.releaseStart()
  820. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGTERM' })
  821. })
  822. it('aborts a stalled preparation request before reporting startup quiescence', async () => {
  823. const fake = new FakeSandbox()
  824. let preparationSignal: AbortSignal | undefined
  825. fake.environmentRequest = async (signal) => {
  826. preparationSignal = signal
  827. await new Promise<never>((_resolve, reject) => {
  828. const rejectAbort = (): void => {
  829. const reason: unknown = signal?.reason
  830. reject(reason instanceof Error ? reason : new Error(String(reason)))
  831. }
  832. if (signal?.aborted === true) {
  833. rejectAbort()
  834. return
  835. }
  836. signal?.addEventListener('abort', rejectAbort, { once: true })
  837. })
  838. }
  839. const handle = testHandle(runtime(fake), spec(), '/runtime/stalled-preparation')
  840. await vi.waitFor(() => { expect(preparationSignal).toBeDefined() })
  841. handle.terminate()
  842. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGTERM' })
  843. await expect(handle.waitForExit()).resolves.toBe(true)
  844. expect(preparationSignal?.aborted).toBe(true)
  845. expect(fake.startOptions).toBeUndefined()
  846. })
  847. it('kills through the provisional SDK handle before process-group publication', async () => {
  848. const fake = new FakeSandbox()
  849. fake.deferProcessGroupRead()
  850. fake.signalErrors.push(commandError(1), commandError(1))
  851. const handle = testHandle(runtime(fake), spec(), '/runtime/pre-publication-kill')
  852. await vi.waitFor(() => { expect(fake.startOptions).toBeDefined() })
  853. handle.terminate()
  854. await vi.waitFor(() => { expect(fake.handle.kills).toBe(1) })
  855. expect(fake.alive).toBe(false)
  856. await expect(handle.waitForExit()).resolves.toBe(true)
  857. fake.releaseProcessGroupRead()
  858. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGKILL' })
  859. })
  860. it('does not treat an unsuccessful SDK fallback as provisional group quiescence', async () => {
  861. const fake = new FakeSandbox()
  862. fake.deferProcessGroupRead()
  863. fake.trapsTerm = true
  864. fake.delaysKill = true
  865. fake.delaysKillCompletion = true
  866. fake.sdkKillStops = false
  867. fake.handle.killResult = false
  868. const handle = testHandle(runtime(fake), spec({ graceMs: 1 }), '/runtime/provisional-sdk-false')
  869. await vi.waitFor(() => { expect(fake.startOptions).toBeDefined() })
  870. handle.terminate()
  871. await vi.waitFor(() => { expect(fake.handle.kills).toBe(1) })
  872. await expect(handle.waitForExit()).rejects.toThrow('remained live after force termination')
  873. fake.alive = false
  874. handle.terminate()
  875. await expect(handle.waitForExit()).resolves.toBe(true)
  876. fake.releaseProcessGroupRead()
  877. fake.finish()
  878. await handle.done
  879. })
  880. it('bounds a quiescence observer while provisional termination is awaiting the controller', async () => {
  881. const fake = new FakeSandbox()
  882. fake.deferProcessGroupRead()
  883. const reconnect = Promise.withResolvers<Sandbox>()
  884. let calls = 0
  885. const delayedRuntime = runtime(fake, async () => {
  886. calls += 1
  887. return calls === 1 ? fake.sandbox : await reconnect.promise
  888. })
  889. const handle = testHandle(delayedRuntime, spec(), '/runtime/pre-publication-observer')
  890. await vi.waitFor(() => { expect(fake.startOptions).toBeDefined() })
  891. handle.terminate()
  892. const controller = new AbortController()
  893. const waiting = handle.waitForExit(controller.signal)
  894. await flush()
  895. controller.abort()
  896. await expect(waiting).resolves.toBe(false)
  897. reconnect.resolve(fake.sandbox)
  898. await expect(handle.waitForExit()).resolves.toBe(true)
  899. fake.releaseProcessGroupRead()
  900. await handle.done
  901. })
  902. it('proves a provisional group exit when the SDK kill fallback fails', async () => {
  903. const fake = new FakeSandbox()
  904. fake.deferProcessGroupRead()
  905. fake.trapsTerm = true
  906. fake.handle.killError = new Error('SDK kill unavailable')
  907. const handle = testHandle(runtime(fake), spec({ graceMs: 1 }), '/runtime/pre-publication-group-kill')
  908. await vi.waitFor(() => { expect(fake.startOptions).toBeDefined() })
  909. handle.terminate()
  910. await expect(handle.waitForExit()).resolves.toBe(true)
  911. fake.releaseProcessGroupRead()
  912. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGKILL' })
  913. })
  914. it('reports failed provisional group and SDK force transports', async () => {
  915. const fake = new FakeSandbox()
  916. fake.deferProcessGroupRead()
  917. fake.signalErrors.push(new Error('TERM transport failed'), new Error('KILL transport failed'))
  918. fake.handle.killError = new Error('SDK kill failed')
  919. const handle = testHandle(runtime(fake), spec({ graceMs: 1 }), '/runtime/pre-publication-failure')
  920. await vi.waitFor(() => { expect(fake.startOptions).toBeDefined() })
  921. handle.terminate()
  922. await expect(handle.waitForExit()).rejects.toThrow('remained live after force termination')
  923. fake.handle.killError = undefined
  924. handle.terminate()
  925. await expect(handle.waitForExit()).resolves.toBe(true)
  926. fake.releaseProcessGroupRead()
  927. await handle.done
  928. const absentGroup = new FakeSandbox()
  929. absentGroup.deferProcessGroupRead()
  930. absentGroup.signalErrors.push(commandError(1), commandError(1))
  931. absentGroup.handle.killError = new Error('SDK kill failed without a provisional group')
  932. const absentHandle = testHandle(
  933. runtime(absentGroup),
  934. spec({ graceMs: 1 }),
  935. '/runtime/pre-publication-absent-group',
  936. )
  937. await vi.waitFor(() => { expect(absentGroup.startOptions).toBeDefined() })
  938. absentHandle.terminate()
  939. await expect(absentHandle.waitForExit()).rejects.toThrow('remained live after force termination')
  940. absentGroup.handle.killError = undefined
  941. absentHandle.terminate()
  942. await expect(absentHandle.waitForExit()).resolves.toBe(true)
  943. absentGroup.releaseProcessGroupRead()
  944. await absentHandle.done
  945. const optimisticSdk = new FakeSandbox()
  946. optimisticSdk.deferProcessGroupRead()
  947. optimisticSdk.signalErrors.push(commandError(1), commandError(1))
  948. optimisticSdk.sdkKillStops = false
  949. const optimisticHandle = testHandle(
  950. runtime(optimisticSdk),
  951. spec({ graceMs: 1 }),
  952. '/runtime/pre-publication-optimistic-sdk',
  953. )
  954. await vi.waitFor(() => { expect(optimisticSdk.startOptions).toBeDefined() })
  955. optimisticHandle.terminate()
  956. await expect(optimisticHandle.waitForExit()).rejects.toThrow('remained live after force termination')
  957. optimisticHandle.terminate()
  958. await expect(optimisticHandle.waitForExit()).resolves.toBe(true)
  959. optimisticSdk.releaseProcessGroupRead()
  960. await optimisticHandle.done
  961. })
  962. it('honors an already-aborted signal when constructing the asynchronous handle directly', async () => {
  963. const fake = new FakeSandbox()
  964. const handle = testHandle(runtime(fake), spec({ signal: AbortSignal.abort('stop') }), '/runtime/pre-aborted')
  965. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGTERM' })
  966. })
  967. it('reacts to a signal that aborts after the remote command has started', async () => {
  968. const fake = new FakeSandbox()
  969. const controller = new AbortController()
  970. const handle = testHandle(runtime(fake), spec({ signal: controller.signal }), '/runtime/live-abort')
  971. await flush()
  972. controller.abort('stop')
  973. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGTERM' })
  974. })
  975. it('can terminate a surviving process group after the command leader settles', async () => {
  976. const fake = new FakeSandbox()
  977. const handle = testHandle(runtime(fake), spec(), '/runtime/surviving-group')
  978. await flush()
  979. await fake.completeOutput()
  980. fake.handle.succeed(0)
  981. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  982. expect(fake.alive).toBe(true)
  983. handle.terminate()
  984. await flush()
  985. const signaled = fake.commandsSeen.includes('kill -TERM -- -4242')
  986. if (!signaled) fake.finish()
  987. await expect(handle.waitForExit()).resolves.toBe(true)
  988. expect(signaled).toBe(true)
  989. })
  990. it('bounds waitForExit while startup or a live group is pending', async () => {
  991. const fake = new FakeSandbox()
  992. fake.deferStart()
  993. const handle = testHandle(runtime(fake), spec(), '/runtime/wait')
  994. const beforeStart = new AbortController()
  995. const pending = handle.waitForExit(beforeStart.signal)
  996. beforeStart.abort()
  997. await expect(pending).resolves.toBe(false)
  998. await expect(handle.waitForExit(AbortSignal.abort())).resolves.toBe(false)
  999. fake.releaseStart()
  1000. await flush()
  1001. const live = new AbortController()
  1002. const liveWait = handle.waitForExit(live.signal)
  1003. live.abort()
  1004. await expect(liveWait).resolves.toBe(false)
  1005. fake.finish()
  1006. await handle.done
  1007. const terminatingFake = new FakeSandbox()
  1008. terminatingFake.deferStart()
  1009. const terminating = testHandle(runtime(terminatingFake), spec(), '/runtime/wait-termination-start')
  1010. terminating.terminate()
  1011. const beforeHandle = new AbortController()
  1012. const handlePending = terminating.waitForExit(beforeHandle.signal)
  1013. beforeHandle.abort()
  1014. await expect(handlePending).resolves.toBe(false)
  1015. terminatingFake.releaseStart()
  1016. await terminating.done
  1017. })
  1018. it('bounds both sides of the liveness-poll abort race', async () => {
  1019. const fake = new FakeSandbox()
  1020. const handle = testHandle(runtime(fake), spec(), '/runtime/poll-abort')
  1021. await flush()
  1022. const beforeTick = new AbortController()
  1023. fake.afterProbe = () => { beforeTick.abort(); fake.afterProbe = undefined }
  1024. await expect(handle.waitForExit(beforeTick.signal)).resolves.toBe(false)
  1025. const duringTick = new AbortController()
  1026. fake.afterProbe = () => {
  1027. fake.afterProbe = undefined
  1028. setTimeout(() => { duringTick.abort() }, 0)
  1029. }
  1030. await expect(handle.waitForExit(duringTick.signal)).resolves.toBe(false)
  1031. const duringProbe = new AbortController()
  1032. fake.beforeProbe = () => { duringProbe.abort(); fake.beforeProbe = undefined }
  1033. await expect(handle.waitForExit(duringProbe.signal)).resolves.toBe(false)
  1034. let racedAbort = false
  1035. const raceSignal = {
  1036. get aborted() { return racedAbort },
  1037. addEventListener: () => { racedAbort = true },
  1038. removeEventListener: () => {},
  1039. } as unknown as AbortSignal
  1040. await expect(handle.waitForExit(raceSignal)).resolves.toBe(false)
  1041. fake.finish()
  1042. await handle.done
  1043. })
  1044. it('observes a live group across one successful bounded poll', async () => {
  1045. const fake = new FakeSandbox()
  1046. const handle = testHandle(runtime(fake), spec(), '/runtime/poll-success')
  1047. await flush()
  1048. setTimeout(() => { fake.finish() }, 1)
  1049. await expect(handle.waitForExit(new AbortController().signal)).resolves.toBe(true)
  1050. await handle.done
  1051. })
  1052. it('treats startup failure as no live tree and contains readiness rejection', async () => {
  1053. const fake = new FakeSandbox()
  1054. fake.backgroundError = new Error('start failed')
  1055. const handle = testHandle(runtime(fake), spec(), '/runtime/fail')
  1056. await expect(handle.done).rejects.toThrow('start failed')
  1057. expect(handle.pid).toBe(-1)
  1058. expect(fake.removed).toContain('/runtime/fail/environment')
  1059. expect(fake.removed).toContain('/runtime/fail')
  1060. await expect(handle.waitForExit()).resolves.toBe(true)
  1061. handle.terminate()
  1062. const unavailableHandle = testHandle(
  1063. runtime(new FakeSandbox(), async () => { throw new Error('sandbox unavailable') }),
  1064. spec(),
  1065. '/runtime/unavailable-start',
  1066. )
  1067. await expect(unavailableHandle.done).rejects.toThrow('sandbox unavailable')
  1068. await expect(unavailableHandle.waitForExit()).resolves.toBe(true)
  1069. const envFailure = new FakeSandbox()
  1070. envFailure.envError = new Error('ambient lookup failed')
  1071. const envHandle = testHandle(runtime(envFailure), spec(), '/runtime/env-failure')
  1072. await expect(envHandle.done).rejects.toThrow('ambient lookup failed')
  1073. expect(envFailure.removed).toEqual([])
  1074. const expectEnvironmentFailure = async (name: string, wire: string, message: string): Promise<void> => {
  1075. const fake = new FakeSandbox()
  1076. fake.environmentWire = wire
  1077. const failed = testHandle(runtime(fake), spec(), `/runtime/${name}`)
  1078. await expect(failed.done).rejects.toThrow(message)
  1079. }
  1080. const encodedEnvironment = Buffer.from('PATH=/bin\0').toString('base64')
  1081. const encodedHome = Buffer.from('/home/user').toString('base64')
  1082. await expectEnvironmentFailure('malformed-frame', '%', 'invalid base64')
  1083. await expectEnvironmentFailure('malformed-base64', `${encodedHome}\n%`, 'invalid base64')
  1084. await expectEnvironmentFailure(
  1085. 'invalid-utf8-home',
  1086. `${Buffer.from([0xff]).toString('base64')}\n${encodedEnvironment}`,
  1087. 'not valid UTF-8',
  1088. )
  1089. await expectEnvironmentFailure(
  1090. 'invalid-utf8-environment',
  1091. `${encodedHome}\n${Buffer.from([0xff]).toString('base64')}`,
  1092. 'not valid UTF-8',
  1093. )
  1094. await expectEnvironmentFailure(
  1095. 'relative-home',
  1096. `${Buffer.from('home/user').toString('base64')}\n${encodedEnvironment}`,
  1097. 'remote login home is invalid',
  1098. )
  1099. await expectEnvironmentFailure(
  1100. 'nul-home',
  1101. `${Buffer.from('/home/user\0tail').toString('base64')}\n${encodedEnvironment}`,
  1102. 'remote login home is invalid',
  1103. )
  1104. const cleanupFailure = new FakeSandbox()
  1105. cleanupFailure.backgroundError = new Error('start failed before credential consumption')
  1106. cleanupFailure.nextRemoveError = new Error('credential cleanup failed')
  1107. const cleanupHandle = testHandle(runtime(cleanupFailure), spec(), '/runtime/cleanup-failure')
  1108. await expect(cleanupHandle.done).rejects.toThrow('command failed and private state cleanup failed')
  1109. const absentState = new FakeSandbox()
  1110. absentState.backgroundError = new Error('start failed after external cleanup')
  1111. absentState.nextRemoveError = new FileNotFoundError('already removed')
  1112. const absentHandle = testHandle(runtime(absentState), spec(), '/runtime/absent-state')
  1113. await expect(absentHandle.done).rejects.toThrow('start failed after external cleanup')
  1114. })
  1115. it('bounds a readiness rejection with a still-live caller signal', async () => {
  1116. const fake = new FakeSandbox()
  1117. fake.deferStart()
  1118. fake.backgroundError = new Error('start failed')
  1119. const handle = testHandle(runtime(fake), spec(), '/runtime/fail-with-signal')
  1120. const waiting = handle.waitForExit(new AbortController().signal)
  1121. fake.releaseStart()
  1122. await expect(handle.done).rejects.toThrow('start failed')
  1123. await expect(waiting).resolves.toBe(true)
  1124. })
  1125. it('propagates an unavailable sandbox unless the caller aborts the wait', async () => {
  1126. const fake = new FakeSandbox()
  1127. let calls = 0
  1128. const unavailable = runtime(fake, async () => {
  1129. calls += 1
  1130. if (calls === 1) return fake.sandbox
  1131. throw new Error('connection unavailable')
  1132. })
  1133. const handle = testHandle(unavailable, spec(), '/runtime/unavailable')
  1134. await flush()
  1135. await expect(handle.waitForExit()).rejects.toThrow('connection unavailable')
  1136. fake.finish()
  1137. await handle.done
  1138. })
  1139. it('returns false when the caller aborts while reconnecting for liveness', async () => {
  1140. const fake = new FakeSandbox()
  1141. const reconnect = Promise.withResolvers<Sandbox>()
  1142. let calls = 0
  1143. const unavailable = runtime(fake, async () => {
  1144. calls += 1
  1145. return calls === 1 ? fake.sandbox : await reconnect.promise
  1146. })
  1147. const handle = testHandle(unavailable, spec(), '/runtime/reconnect-abort')
  1148. await flush()
  1149. const controller = new AbortController()
  1150. const waiting = handle.waitForExit(controller.signal)
  1151. await flush()
  1152. controller.abort()
  1153. reconnect.reject(new Error('connection unavailable'))
  1154. await expect(waiting).resolves.toBe(false)
  1155. fake.finish()
  1156. await handle.done
  1157. })
  1158. it('returns false when a liveness request itself is aborted and surfaces other probe failures', async () => {
  1159. const fake = new FakeSandbox()
  1160. const handle = testHandle(runtime(fake), spec(), '/runtime/probe')
  1161. await flush()
  1162. const controller = new AbortController()
  1163. controller.abort()
  1164. await expect(handle.waitForExit(controller.signal)).resolves.toBe(false)
  1165. fake.probeError = new Error('probe failed')
  1166. await expect(handle.waitForExit()).rejects.toThrow('probe failed')
  1167. fake.finish()
  1168. await handle.done
  1169. })
  1170. it('treats a timeout-killed sandbox as quiescent during liveness probing', async () => {
  1171. const fake = new FakeSandbox()
  1172. const handle = testHandle(runtime(fake), spec(), '/runtime/expired-sandbox')
  1173. await flush()
  1174. fake.finish()
  1175. await handle.done
  1176. fake.probeError = new SandboxNotFoundError('sandbox expired')
  1177. await expect(handle.waitForExit()).resolves.toBe(true)
  1178. })
  1179. it('treats a missing sandbox handle as quiescent during liveness acquisition', async () => {
  1180. const fake = new FakeSandbox()
  1181. let calls = 0
  1182. const handle = testHandle(runtime(fake, async () => {
  1183. calls += 1
  1184. if (calls === 1) return fake.sandbox
  1185. throw new SandboxNotFoundError('sandbox expired')
  1186. }), spec(), '/runtime/expired-acquisition')
  1187. await flush()
  1188. await expect(handle.waitForExit()).resolves.toBe(true)
  1189. await fake.completeOutput()
  1190. fake.alive = false
  1191. fake.handle.succeed(0)
  1192. await handle.done
  1193. })
  1194. it('treats sandbox loss during termination as quiescent', async () => {
  1195. const fake = new FakeSandbox()
  1196. let calls = 0
  1197. const handle = testHandle(runtime(fake, async () => {
  1198. calls += 1
  1199. if (calls === 1) return fake.sandbox
  1200. throw new SandboxNotFoundError('sandbox expired')
  1201. }), spec(), '/runtime/expired-termination')
  1202. await flush()
  1203. await fake.completeOutput()
  1204. handle.terminate()
  1205. await expect(handle.waitForExit()).resolves.toBe(true)
  1206. fake.alive = false
  1207. fake.handle.succeed(0)
  1208. await handle.done
  1209. })
  1210. it('makes batch stdin close failures best-effort', async () => {
  1211. const fake = new FakeSandbox()
  1212. vi.spyOn(fake.handle, 'sendStdin').mockRejectedValueOnce(new Error('closed'))
  1213. const handle = testHandle(runtime(fake), spec({
  1214. stdio: { stdin: { data: 'ignored' }, stdout: { maxBytes: 4 }, stderr: { maxBytes: 4 } },
  1215. }), '/runtime/stdin-closed')
  1216. await flush()
  1217. fake.finish()
  1218. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  1219. })
  1220. it('rejects malformed SDK process ids and non-command settlement failures', async () => {
  1221. const invalidPid = new FakeSandbox()
  1222. invalidPid.handle.pid = 0
  1223. const invalid = testHandle(runtime(invalidPid), spec(), '/runtime/invalid-pid')
  1224. await expect(invalid.done).rejects.toThrow(/invalid command pid 0/)
  1225. expect(invalidPid.handle.kills).toBe(1)
  1226. expect(invalidPid.removed).toContain('/runtime/invalid-pid/environment')
  1227. await expect(invalid.waitForExit()).resolves.toBe(true)
  1228. const failedRollback = new FakeSandbox()
  1229. failedRollback.handle.pid = 0
  1230. failedRollback.handle.killError = new Error('invalid handle kill failed')
  1231. const retained = testHandle(runtime(failedRollback), spec(), '/runtime/invalid-pid-retained')
  1232. await expect(retained.done).rejects.toThrow('invalid command pid rollback did not reach quiescence')
  1233. await expect(retained.waitForExit()).rejects.toThrow('invalid handle kill failed')
  1234. failedRollback.handle.killError = undefined
  1235. retained.terminate()
  1236. await expect(retained.waitForExit()).resolves.toBe(true)
  1237. const crashedFake = new FakeSandbox()
  1238. const crashed = testHandle(runtime(crashedFake), spec(), '/runtime/crashed')
  1239. await flush()
  1240. crashedFake.alive = false
  1241. crashedFake.handle.crash(new Error('command transport failed'))
  1242. await expect(crashed.done).rejects.toThrow('command transport failed')
  1243. })
  1244. it('rejects invalid or absent process-group publication', async () => {
  1245. const invalidGroup = new FakeSandbox()
  1246. invalidGroup.processGroupId = 'not-a-pid\n'
  1247. invalidGroup.delaysKill = true
  1248. invalidGroup.sdkKillStops = false
  1249. invalidGroup.afterProbe = () => { invalidGroup.alive = false }
  1250. const invalid = testHandle(runtime(invalidGroup), spec(), '/runtime/invalid-group')
  1251. await expect(invalid.done).rejects.toThrow(/invalid process-group id/)
  1252. expect(invalidGroup.handle.kills).toBe(1)
  1253. expect(invalidGroup.commandsSeen).toContain('kill -KILL -- -4242')
  1254. await expect(invalid.waitForExit()).resolves.toBe(true)
  1255. // A rewritten pid file must not aim the kill at every process (`-- -1`).
  1256. const unsafeGroup = new FakeSandbox()
  1257. unsafeGroup.processGroupId = '1\n'
  1258. unsafeGroup.delaysKill = true
  1259. unsafeGroup.sdkKillStops = false
  1260. unsafeGroup.afterProbe = () => { unsafeGroup.alive = false }
  1261. const unsafe = testHandle(runtime(unsafeGroup), spec(), '/runtime/unsafe-group')
  1262. await expect(unsafe.done).rejects.toThrow(/unsafe published process-group id 1/)
  1263. expect(unsafeGroup.commandsSeen).not.toContain('kill -KILL -- -1')
  1264. await expect(unsafe.waitForExit()).resolves.toBe(true)
  1265. const absentGroup = new FakeSandbox()
  1266. absentGroup.processGroupId = ''
  1267. const absent = testHandle(runtime(absentGroup), spec(), '/runtime/absent-group')
  1268. await flush()
  1269. absentGroup.finish()
  1270. await expect(absent.done).rejects.toThrow(/exited before publishing/)
  1271. expect(absentGroup.handle.kills).toBe(1)
  1272. expect(absentGroup.commandsSeen).toContain('kill -KILL -- -4242')
  1273. await expect(absent.waitForExit()).resolves.toBe(true)
  1274. })
  1275. it('preserves publication failure and reports cleanup that cannot be verified', async () => {
  1276. const fake = new FakeSandbox()
  1277. fake.processGroupId = 'not-a-pid\n'
  1278. fake.signalError = new Error('rollback signal failed')
  1279. fake.handle.killError = new Error('SDK kill failed')
  1280. const handle = testHandle(runtime(fake), spec(), '/runtime/failed-rollback')
  1281. let failure: unknown
  1282. try {
  1283. await handle.done
  1284. } catch (error: unknown) {
  1285. failure = error
  1286. }
  1287. expect(failure).toBeInstanceOf(AggregateError)
  1288. if (!(failure instanceof AggregateError)) throw new Error('expected AggregateError')
  1289. expect(failure.message).toBe('subprocess-e2b: process-group publication failed and rollback did not reach quiescence')
  1290. const failures = Array.from(failure.errors as Iterable<unknown>)
  1291. expect(failures).toHaveLength(2)
  1292. expect(failures[0]).toBeInstanceOf(Error)
  1293. expect(failures[1]).toBeInstanceOf(Error)
  1294. if (!(failures[0] instanceof Error) || !(failures[1] instanceof Error)) throw new Error('expected nested errors')
  1295. expect(failures[0].message).toContain('invalid process-group id')
  1296. expect(failures[1].message).toContain('remained live after force termination')
  1297. expect(fake.handle.kills).toBe(1)
  1298. const bounded = new AbortController()
  1299. const waiting = handle.waitForExit(bounded.signal)
  1300. bounded.abort()
  1301. await expect(waiting).resolves.toBe(false)
  1302. handle.terminate()
  1303. await expect(handle.waitForExit()).resolves.toBe(true)
  1304. expect(fake.commandsSeen).toContain('kill -TERM -- -4242')
  1305. const naturallyGone = new FakeSandbox()
  1306. naturallyGone.processGroupId = 'not-a-pid\n'
  1307. naturallyGone.signalError = new Error('rollback signal failed')
  1308. naturallyGone.handle.killError = new Error('SDK kill failed')
  1309. const observed = testHandle(runtime(naturallyGone), spec(), '/runtime/failed-rollback-observed')
  1310. await expect(observed.done).rejects.toThrow('process-group publication failed')
  1311. naturallyGone.alive = false
  1312. await expect(observed.waitForExit()).resolves.toBe(true)
  1313. })
  1314. it('waits for delayed process-group publication', async () => {
  1315. const fake = new FakeSandbox()
  1316. fake.processGroupReads.push('', '4242\n')
  1317. const handle = testHandle(runtime(fake), spec(), '/runtime/delayed-group')
  1318. await vi.waitFor(() => { expect(handle.pid).toBe(4242) })
  1319. fake.finish()
  1320. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  1321. })
  1322. it('handles output backpressure and contains a stderr sink failure', async () => {
  1323. const fake = new FakeSandbox()
  1324. const handle = testHandle(runtime(fake), spec({
  1325. stdio: { stdin: 'ignore', stdout: 'pipe', stderr: 'pipe' },
  1326. }), '/runtime/backpressure')
  1327. await flush()
  1328. handle.stdout!.on('error', () => {})
  1329. const stdoutWrite = vi.spyOn(handle.stdout!, 'write').mockReturnValueOnce(false)
  1330. const stdoutPending = fake.stdout('blocked')
  1331. queueMicrotask(() => { handle.stdout!.emit('drain') })
  1332. await stdoutPending
  1333. stdoutWrite.mockRestore()
  1334. handle.stderr!.on('error', () => {})
  1335. const stderrWrite = vi.spyOn(handle.stderr!, 'write').mockReturnValueOnce(false)
  1336. const stderrPending = fake.stderr('broken')
  1337. queueMicrotask(() => { handle.stderr!.emit('error', new Error('sink failed')) })
  1338. await stderrPending
  1339. stderrWrite.mockRestore()
  1340. fake.finish()
  1341. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  1342. })
  1343. it('settles output backpressure when the consumer closes the pipe', async () => {
  1344. const fake = new FakeSandbox()
  1345. const handle = testHandle(runtime(fake), spec({
  1346. stdio: { stdin: 'ignore', stdout: 'pipe', stderr: { maxBytes: 4 } },
  1347. }), '/runtime/backpressure-close')
  1348. await flush()
  1349. const stdoutWrite = vi.spyOn(handle.stdout!, 'write').mockReturnValueOnce(false)
  1350. const pending = fake.stdout('discarded')
  1351. queueMicrotask(() => { handle.stdout!.destroy() })
  1352. await pending
  1353. stdoutWrite.mockRestore()
  1354. fake.finish()
  1355. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  1356. })
  1357. it('breaks output backpressure when termination owns the command', async () => {
  1358. const fake = new FakeSandbox()
  1359. const handle = testHandle(runtime(fake), spec({
  1360. stdio: { stdin: 'ignore', stdout: 'pipe', stderr: { maxBytes: 4 } },
  1361. }), '/runtime/backpressure-termination')
  1362. await flush()
  1363. const stdoutWrite = vi.spyOn(handle.stdout!, 'write').mockReturnValueOnce(false)
  1364. let released = false
  1365. const pending = fake.stdout('blocked').then(() => { released = true })
  1366. await Promise.resolve()
  1367. handle.terminate()
  1368. await flush()
  1369. const releasedByTermination = released
  1370. if (!released) handle.stdout!.emit('drain')
  1371. await pending
  1372. stdoutWrite.mockRestore()
  1373. await handle.done
  1374. expect(releasedByTermination).toBe(true)
  1375. })
  1376. it('settles backpressure when a synchronous pipe write starts termination', async () => {
  1377. const fake = new FakeSandbox()
  1378. const handle = testHandle(runtime(fake), spec({
  1379. stdio: { stdin: 'ignore', stdout: 'pipe', stderr: { maxBytes: 4 } },
  1380. }), '/runtime/backpressure-synchronous-termination')
  1381. await flush()
  1382. const stdoutWrite = vi.spyOn(handle.stdout!, 'write').mockImplementationOnce(() => {
  1383. handle.terminate()
  1384. return false
  1385. })
  1386. await expect(fake.stdout('blocked')).resolves.toBeUndefined()
  1387. stdoutWrite.mockRestore()
  1388. await handle.done
  1389. })
  1390. it('contains a pipe callback failure instead of rejecting command settlement', async () => {
  1391. const fake = new FakeSandbox()
  1392. const handle = testHandle(runtime(fake), spec({
  1393. stdio: { stdin: 'ignore', stdout: 'pipe', stderr: { maxBytes: 4 } },
  1394. }), '/runtime/pipe-error')
  1395. await flush()
  1396. const emitted = once(handle.stdout!, 'error')
  1397. handle.stdout!.destroy(new Error('consumer failed'))
  1398. await emitted
  1399. await fake.stdout('late output')
  1400. fake.finish()
  1401. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  1402. })
  1403. it('contains an already-gone group signal and escalates after a TERM transport failure', async () => {
  1404. const gone = new FakeSandbox()
  1405. gone.trapsTerm = true
  1406. gone.signalError = commandError(1)
  1407. const goneHandle = testHandle(runtime(gone), spec({ graceMs: 1 }), '/runtime/gone-signal')
  1408. await flush()
  1409. goneHandle.terminate()
  1410. await expect(goneHandle.done).resolves.toEqual({ exitCode: null, signal: 'SIGKILL' })
  1411. const failed = new FakeSandbox()
  1412. failed.signalError = new Error('signal transport failed')
  1413. const failedHandle = testHandle(runtime(failed), spec(), '/runtime/failed-signal')
  1414. await flush()
  1415. failedHandle.terminate()
  1416. await expect(failedHandle.done).resolves.toEqual({ exitCode: null, signal: 'SIGKILL' })
  1417. expect(failed.commandsSeen).toContain('kill -KILL -- -4242')
  1418. })
  1419. it('allows termination retry after both force transports fail', async () => {
  1420. const fake = new FakeSandbox()
  1421. fake.signalErrors.push(new Error('TERM transport failed'), new Error('KILL transport failed'))
  1422. fake.handle.killError = new Error('SDK kill failed')
  1423. const handle = testHandle(runtime(fake), spec({ graceMs: 1 }), '/runtime/retry-signal')
  1424. await flush()
  1425. handle.terminate()
  1426. await expect(handle.waitForExit()).rejects.toThrow('remained live after force termination')
  1427. fake.handle.killError = undefined
  1428. handle.terminate()
  1429. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGTERM' })
  1430. expect(fake.commandsSeen.filter(command => command.startsWith('kill -TERM '))).toHaveLength(2)
  1431. const missingGroup = new FakeSandbox()
  1432. missingGroup.trapsTerm = true
  1433. missingGroup.signalErrors.push(undefined, commandError(1))
  1434. missingGroup.handle.killError = new Error('SDK kill failed after group exit race')
  1435. const raced = testHandle(runtime(missingGroup), spec({ graceMs: 1 }), '/runtime/group-exit-race')
  1436. await flush()
  1437. raced.terminate()
  1438. await expect(raced.waitForExit()).rejects.toThrow('remained live after force termination')
  1439. missingGroup.handle.killError = undefined
  1440. raced.terminate()
  1441. await expect(raced.waitForExit()).resolves.toBe(true)
  1442. })
  1443. it('rejects an optimistic SDK kill while descendants survive a failed group KILL', async () => {
  1444. const fake = new FakeSandbox()
  1445. fake.trapsTerm = true
  1446. fake.sdkKillStops = false
  1447. fake.signalErrors.push(undefined, new Error('KILL transport failed'))
  1448. const handle = testHandle(runtime(fake), spec({ graceMs: 1 }), '/runtime/optimistic-sdk-kill')
  1449. await flush()
  1450. handle.terminate()
  1451. await expect(handle.waitForExit()).rejects.toThrow('remained live after force termination')
  1452. expect(fake.alive).toBe(true)
  1453. fake.sdkKillStops = true
  1454. handle.terminate()
  1455. await expect(handle.waitForExit()).resolves.toBe(true)
  1456. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGKILL' })
  1457. })
  1458. })
  1459. describe('E2BSubprocessRuntime', () => {
  1460. async function service(
  1461. fake = new FakeSandbox(),
  1462. providedRuntime: E2BRuntime = runtime(fake),
  1463. ): Promise<{ ctx: Context; fiber: Awaited<ReturnType<Context['plugin']>> }> {
  1464. const ctx = new Context()
  1465. ctx.provide('e2b', providedRuntime)
  1466. const fiber = await ctx.plugin(E2BSubprocessRuntime)
  1467. return { ctx, fiber }
  1468. }
  1469. it('registers handles and disposal terminates and joins live remote groups regardless of sandbox policy', async () => {
  1470. const fake = new FakeSandbox()
  1471. fake.trapsTerm = true
  1472. const { ctx, fiber } = await service(fake)
  1473. const handle = ctx.subprocess.spawn(spec({ graceMs: 1 }))
  1474. await flush()
  1475. await fiber.dispose()
  1476. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGKILL' })
  1477. expect(fake.alive).toBe(false)
  1478. })
  1479. it('awaits SDK settlement after the remote process group becomes quiescent', async () => {
  1480. const fake = new FakeSandbox()
  1481. fake.trapsTerm = true
  1482. const { ctx, fiber } = await service(fake)
  1483. const handle = ctx.subprocess.spawn(spec())
  1484. await flush()
  1485. fake.alive = false
  1486. let disposed = false
  1487. const disposing = fiber.dispose().then(() => { disposed = true })
  1488. await flush()
  1489. expect(disposed).toBe(false)
  1490. fake.finish()
  1491. await disposing
  1492. await expect(handle.done).resolves.toEqual({ exitCode: 0, signal: null })
  1493. })
  1494. it('reports a failed termination transaction from disposal instead of waiting on done', async () => {
  1495. const fake = new FakeSandbox()
  1496. fake.signalErrors.push(new Error('TERM transport failed'), new Error('KILL transport failed'))
  1497. fake.handle.killError = new Error('SDK kill failed')
  1498. const { ctx, fiber } = await service(fake)
  1499. const handle = ctx.subprocess.spawn(spec({ graceMs: 1 }))
  1500. await flush()
  1501. await expect(fiber.dispose()).resolves.toBeUndefined()
  1502. await expect(handle.waitForExit()).rejects.toThrow('remained live after force termination')
  1503. fake.handle.killError = undefined
  1504. handle.terminate()
  1505. await expect(handle.waitForExit()).resolves.toBe(true)
  1506. await expect(handle.done).resolves.toEqual({ exitCode: null, signal: 'SIGTERM' })
  1507. })
  1508. it('aggregates sibling cleanup failures instead of reporting only the first', async () => {
  1509. const { ctx, fiber } = await service()
  1510. const disposalErrors: unknown[] = []
  1511. ctx.logger.error = ((error: unknown) => { disposalErrors.push(error) }) as typeof ctx.logger.error
  1512. const first = {
  1513. terminate: vi.fn(),
  1514. waitForExit: vi.fn(async () => { throw new Error('first cleanup failed') }),
  1515. done: Promise.resolve({ exitCode: 0, signal: null }),
  1516. } as unknown as E2BSubprocessHandle
  1517. const second = {
  1518. terminate: vi.fn(),
  1519. waitForExit: vi.fn(async () => { throw new Error('second cleanup failed') }),
  1520. done: Promise.resolve({ exitCode: 0, signal: null }),
  1521. } as unknown as E2BSubprocessHandle
  1522. const live = (ctx.subprocess as unknown as { live: Set<E2BSubprocessHandle> }).live
  1523. live.add(first)
  1524. live.add(second)
  1525. await fiber.dispose()
  1526. const failure = disposalErrors[0]
  1527. expect(failure).toBeInstanceOf(AggregateError)
  1528. if (!(failure instanceof AggregateError)) throw new Error('expected AggregateError')
  1529. expect(failure.errors.map(error => (error as Error).message).sort()).toEqual([
  1530. 'first cleanup failed',
  1531. 'second cleanup failed',
  1532. ])
  1533. })
  1534. it('waits for every owned cleanup before reporting a disposal failure', async () => {
  1535. const { ctx, fiber } = await service()
  1536. const failed = {
  1537. terminate: vi.fn(),
  1538. waitForExit: vi.fn(async () => { throw new Error('cleanup failed') }),
  1539. done: Promise.resolve({ exitCode: 0, signal: null }),
  1540. } as unknown as E2BSubprocessHandle
  1541. let finishCleanup!: () => void
  1542. const cleanup = new Promise<boolean>((resolve) => {
  1543. finishCleanup = () => { resolve(true) }
  1544. })
  1545. const draining = {
  1546. terminate: vi.fn(),
  1547. waitForExit: vi.fn(() => cleanup),
  1548. done: Promise.resolve({ exitCode: 0, signal: null }),
  1549. } as unknown as E2BSubprocessHandle
  1550. const live = (ctx.subprocess as unknown as { live: Set<E2BSubprocessHandle> }).live
  1551. live.add(failed)
  1552. live.add(draining)
  1553. let disposed = false
  1554. const disposing = fiber.dispose().then(() => { disposed = true })
  1555. await flush()
  1556. expect(disposed).toBe(false)
  1557. finishCleanup()
  1558. await disposing
  1559. expect(live).toEqual(new Set([failed]))
  1560. })
  1561. it('releases naturally settled handles before later service disposal', async () => {
  1562. const fake = new FakeSandbox()
  1563. const { ctx, fiber } = await service(fake)
  1564. const handle = ctx.subprocess.spawn(spec())
  1565. await flush()
  1566. fake.finish()
  1567. await handle.done
  1568. await flush()
  1569. const signalsBefore = fake.commandsSeen.filter(command => command.startsWith('kill -')).length
  1570. await fiber.dispose()
  1571. expect(fake.commandsSeen.filter(command => command.startsWith('kill -')).length).toBe(signalsBefore)
  1572. })
  1573. it('contains a release liveness failure and retries quiescence during disposal', async () => {
  1574. const fake = new FakeSandbox()
  1575. let calls = 0
  1576. const reconnecting = runtime(fake, async () => {
  1577. calls += 1
  1578. if (calls === 2) throw new Error('transient liveness failure')
  1579. return fake.sandbox
  1580. })
  1581. const { ctx, fiber } = await service(fake, reconnecting)
  1582. const handle = ctx.subprocess.spawn(spec())
  1583. await flush()
  1584. fake.finish()
  1585. await handle.done
  1586. await flush()
  1587. await fiber.dispose()
  1588. expect(calls).toBeGreaterThanOrEqual(3)
  1589. })
  1590. it('contains spawn rejection while disposal is joining the pending handle', async () => {
  1591. const fake = new FakeSandbox()
  1592. fake.deferStart()
  1593. fake.backgroundError = new Error('start failed during disposal')
  1594. const { ctx, fiber } = await service(fake)
  1595. const subprocess = ctx.subprocess
  1596. const handle = subprocess.spawn(spec())
  1597. await vi.waitFor(() => { expect(fake.startOptions).toBeDefined() })
  1598. const disposing = fiber.dispose()
  1599. await flush()
  1600. expect(() => subprocess.spawn(spec())).toThrow('service is disposing')
  1601. fake.releaseStart()
  1602. await expect(disposing).resolves.toBeUndefined()
  1603. await expect(handle.done).rejects.toThrow('start failed during disposal')
  1604. })
  1605. it('validates synchronous spawn preconditions', async () => {
  1606. const { ctx } = await service()
  1607. expect(() => ctx.subprocess.spawn(spec({ argv: [] }))).toThrow(/non-empty program/)
  1608. expect(() => ctx.subprocess.spawn(spec({ signal: AbortSignal.abort('stop') }))).toThrow(/aborted before spawn/)
  1609. })
  1610. it('registers the package-owned empty invariant installer', async () => {
  1611. const ctx = new Context()
  1612. await ctx.plugin(InvariantRegistry, { enabled: true })
  1613. const fiber = await ctx.plugin(E2BSubprocessInvariant).await()
  1614. await fiber.dispose()
  1615. })
  1616. })