code-mode.spec.ts 54 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204
  1. import { describe, expect, it } from 'vitest'
  2. import { Context } from 'cordis'
  3. import { CallId } from '@deepseek-ai/dsh-llm'
  4. import { createScope } from '@deepseek-ai/dsh-scope'
  5. import type { Scope } from '@deepseek-ai/dsh-scope'
  6. import SystemPrompt from '@deepseek-ai/dsh-system-prompt'
  7. import { CodeRuntime } from '@deepseek-ai/dsh-code-runtime'
  8. import type { CodeRunRequest, CodeRunResult } from '@deepseek-ai/dsh-code-runtime'
  9. import ToolRegistry, { CodeRunFailedError, RUN_CODE_NAME, TOOL_ABORTED_BEFORE_DISPATCH, defineContentToolFixture, defineTool } from '@deepseek-ai/dsh-tools'
  10. import type { Config, JsonSchemaNode, PostToolDecision, ToolExecutionResult } from '@deepseek-ai/dsh-tools'
  11. import type { Agent } from '@deepseek-ai/dsh-agent'
  12. import { Session, SessionId } from '@deepseek-ai/dsh-session'
  13. import type { JsonValue, SessionEventMap } from '@deepseek-ai/dsh-session'
  14. const testToolSignal = new AbortController().signal
  15. /**
  16. * Code Mode unit tier (per the Agent Note's plan): provider contribution per mode,
  17. * misconfiguration rejections, the run_code dispatch bridge (serialization,
  18. * abort, JSON normalization, error mapping, events, quiescence), and HMR
  19. * safety — all against an in-repo fake runtime, exactly the
  20. * interface/implementation/consumer shape the seam promises.
  21. */
  22. /** A scriptable in-repo CodeRuntime: each test sets `behavior` to drive the bindings however it needs. */
  23. class FakeRuntime extends CodeRuntime {
  24. readonly language: string
  25. readonly isolation = 'fake'
  26. behavior: (request: CodeRunRequest) => Promise<CodeRunResult> = () => Promise.resolve({ logs: [] })
  27. lastRequest?: CodeRunRequest
  28. constructor(ctx: Context, config: { language?: string } = {}) {
  29. super(ctx)
  30. this.language = config.language ?? 'typescript'
  31. }
  32. run(request: CodeRunRequest): Promise<CodeRunResult> {
  33. this.lastRequest = request
  34. return this.behavior(request)
  35. }
  36. }
  37. interface SetupOptions {
  38. mode?: Config['mode']
  39. maxParallelSubCalls?: number
  40. runtime?: false | { language?: string }
  41. toolOrder?: string[]
  42. }
  43. async function setup(options: SetupOptions = {}) {
  44. const ctx = new Context()
  45. await ctx.plugin(SystemPrompt, { ...options.toolOrder ? { toolOrder: options.toolOrder } : {} })
  46. await ctx.plugin(ToolRegistry, { mode: options.mode ?? 'code', ...options.maxParallelSubCalls !== undefined ? { maxParallelSubCalls: options.maxParallelSubCalls } : {} })
  47. let runtime: FakeRuntime | undefined
  48. if (options.runtime !== false) {
  49. await ctx.plugin(FakeRuntime, options.runtime ?? {})
  50. runtime = ctx.codeRuntime as FakeRuntime
  51. }
  52. return { ctx, tools: ctx.tools, systemPrompt: ctx.systemPrompt, runtime: runtime! }
  53. }
  54. /** Mint one production-shaped agent scope that can register scoped tool policy. */
  55. async function mintAgentScope(ctx: Context, name = 'scoped'): Promise<{ scope: Scope; agent: Agent }> {
  56. const agent = { id: SessionId(name) } as Agent
  57. let scope!: Scope
  58. await ctx.plugin(Object.assign((inner: Context) => { scope = createScope(inner, agent) },
  59. { inject: ['tools', 'systemPrompt'] }))
  60. return { scope, agent }
  61. }
  62. /** Register a trivial echo tool; returns the calls it received. */
  63. function registerEcho(ctx: Context, name = 'echo'): unknown[] {
  64. const calls: unknown[] = []
  65. ctx.tools.register(defineTool({
  66. name,
  67. description: `Echo tool ${name}.`,
  68. parameters: { value: { type: 'string', required: true } },
  69. output: {
  70. schema: { type: 'string' },
  71. render: (_args, value) => [{ type: 'text', text: value }],
  72. },
  73. execute(args) {
  74. calls.push(args)
  75. return Promise.resolve(`${name}:${args.value}`)
  76. },
  77. }))
  78. return calls
  79. }
  80. /** A structural fake of the owning agent: captures session appends. */
  81. function fakeAgent(): { agent: Agent; events: { type: string; data: unknown }[] } {
  82. const events: { type: string; data: unknown }[] = []
  83. const agent = {
  84. session: {
  85. header: { cwd: '/workspace' },
  86. append: (type: string, data: unknown) => { events.push({ type, data }) },
  87. },
  88. } as unknown as Agent
  89. return { agent, events }
  90. }
  91. /** Dispatch run_code through the registry pipeline, as the loop would. */
  92. async function runCode(
  93. ctx: Context,
  94. code: string,
  95. extras: { agent?: Agent; signal?: AbortSignal; description?: string } = {},
  96. ): Promise<ToolExecutionResult> {
  97. return ctx.tools.execute({
  98. signal: testToolSignal,
  99. callId: CallId('call-1'),
  100. name: RUN_CODE_NAME,
  101. arguments: { code, description: extras.description ?? 'Run the test program' },
  102. ...extras.agent ? { agent: extras.agent } : {},
  103. ...extras.signal ? { signal: extras.signal } : {},
  104. })
  105. }
  106. describe('mode-aware wire contribution', () => {
  107. it("mode 'native' contributes every schema, no run_code, no SDK section — and needs no runtime", async () => {
  108. const { ctx, systemPrompt } = await setup({ mode: 'native', runtime: false })
  109. registerEcho(ctx)
  110. const assembly = await systemPrompt.assemble()
  111. expect(assembly.tools.map(tool => tool.name)).toEqual(['echo'])
  112. expect(assembly.sections.some(section => section.name === 'tools:sdk')).toBe(false)
  113. })
  114. it("mode 'code' contributes exactly [run_code] plus the SDK section declaring the other tools", async () => {
  115. const { ctx, systemPrompt } = await setup({ mode: 'code' })
  116. registerEcho(ctx)
  117. const assembly = await systemPrompt.assemble()
  118. expect(assembly.tools.map(tool => tool.name)).toEqual([RUN_CODE_NAME])
  119. const sdk = assembly.sections.find(section => section.name === 'tools:sdk')
  120. expect(sdk?.text).toContain('declare const tools: {')
  121. expect(sdk?.text).toContain('echo: {')
  122. expect(sdk?.text).not.toContain('run_code:')
  123. })
  124. it('projects deeply nested output schemas into the Code Mode SDK without structured-clone recursion', async () => {
  125. const { ctx, systemPrompt } = await setup({ mode: 'code' })
  126. let output: JsonSchemaNode = { type: 'string' }
  127. for (let depth = 0; depth < 5_000; depth++) {
  128. output = { oneOf: [output, { type: 'null' }] }
  129. }
  130. ctx.tools.register({
  131. name: 'deep_output',
  132. description: 'Return a deeply nested output union.',
  133. parameters: { type: 'object', properties: {} },
  134. output: {
  135. schema: output,
  136. render: (_args, value) => [{ type: 'text', text: typeof value === 'string' ? value : 'null' }],
  137. },
  138. execute() { return Promise.resolve('ok') },
  139. })
  140. const assembly = await systemPrompt.assemble()
  141. const sdk = assembly.sections.find(section => section.name === 'tools:sdk')?.text
  142. expect(sdk).toContain('deep_output: Record<string, JsonValue>;')
  143. expect(sdk).toContain('deep_output: string | null')
  144. })
  145. it.each(['code', 'both'] as const)('treats expert assembly output as authoritative in mode %s', async (mode) => {
  146. const { ctx, systemPrompt } = await setup({ mode })
  147. registerEcho(ctx)
  148. ctx.on('system-prompt/assemble', async (_assembly, _context, next) => {
  149. const assembly = await next()
  150. return {
  151. ...assembly,
  152. sections: assembly.sections.filter(section => section.name !== 'tools:sdk'),
  153. tools: assembly.tools.filter(tool => tool.name !== RUN_CODE_NAME),
  154. }
  155. }, { prepend: true })
  156. const assembly = await systemPrompt.assemble()
  157. expect(assembly.sections.some(section => section.name === 'tools:sdk')).toBe(false)
  158. expect(assembly.tools.some(tool => tool.name === RUN_CODE_NAME)).toBe(false)
  159. })
  160. it.each(['code', 'both'] as const)('lets one scope shadow the default SDK section in mode %s', async (mode) => {
  161. const { ctx, systemPrompt } = await setup({ mode })
  162. registerEcho(ctx)
  163. const { scope, agent } = await mintAgentScope(ctx)
  164. scope.ctx.systemPrompt.section({ name: 'tools:sdk', order: 150, text: 'SCOPED SDK' })
  165. const scoped = await systemPrompt.assemble({ scope: agent })
  166. const global = await systemPrompt.assemble()
  167. expect(scoped.sections.find(section => section.name === 'tools:sdk')?.text).toBe('SCOPED SDK')
  168. expect(global.sections.find(section => section.name === 'tools:sdk')?.text).toContain('declare const tools:')
  169. })
  170. it("mode 'both' contributes every native schema plus run_code, and the SDK section", async () => {
  171. const { ctx, systemPrompt } = await setup({ mode: 'both' })
  172. registerEcho(ctx)
  173. const assembly = await systemPrompt.assemble()
  174. expect(assembly.tools.map(tool => tool.name)).toEqual(['echo', RUN_CODE_NAME])
  175. expect(assembly.sections.some(section => section.name === 'tools:sdk')).toBe(true)
  176. })
  177. it.each(['code', 'both'] as const)('keeps the run_code transport outside scoped allow-list filtering in mode %s', async (mode) => {
  178. const { ctx, systemPrompt, runtime } = await setup({ mode })
  179. registerEcho(ctx, 'echo')
  180. registerEcho(ctx, 'hidden')
  181. const { scope, agent } = await mintAgentScope(ctx)
  182. const lift = scope.ctx.tools.restrict({ allow: ['echo'] })
  183. const assembly = await systemPrompt.assemble({ scope: agent })
  184. expect(assembly.tools.map(tool => tool.name)).toEqual(mode === 'code'
  185. ? [RUN_CODE_NAME]
  186. : ['echo', RUN_CODE_NAME])
  187. const sdk = assembly.sections.find(section => section.name === 'tools:sdk')?.text
  188. expect(sdk).toContain('echo: {')
  189. expect(sdk).not.toContain('hidden:')
  190. runtime.behavior = request => Promise.resolve({
  191. logs: [],
  192. value: Object.keys(request.bindings[0]!.functions).sort().join(','),
  193. })
  194. const result = await runCode(ctx, 'return Object.keys(tools)', { agent })
  195. expect(result.isError).toBe(false)
  196. expect(result.content).toEqual([{ type: 'text', text: 'echo' }])
  197. lift()
  198. const unrestricted = await systemPrompt.assemble({ scope: agent })
  199. expect(unrestricted.tools.map(tool => tool.name)).toEqual(mode === 'code'
  200. ? [RUN_CODE_NAME]
  201. : ['echo', 'hidden', RUN_CODE_NAME])
  202. })
  203. it.each(['code', 'both'] as const)('keeps the run_code transport outside scoped deny-list filtering in mode %s', async (mode) => {
  204. const { ctx, systemPrompt, runtime } = await setup({ mode })
  205. registerEcho(ctx, 'denied')
  206. registerEcho(ctx, 'kept')
  207. const { scope, agent } = await mintAgentScope(ctx)
  208. scope.ctx.tools.restrict({ deny: ['denied'] })
  209. const assembly = await systemPrompt.assemble({ scope: agent })
  210. expect(assembly.tools.map(tool => tool.name)).toEqual(mode === 'code'
  211. ? [RUN_CODE_NAME]
  212. : ['kept', RUN_CODE_NAME])
  213. const sdk = assembly.sections.find(section => section.name === 'tools:sdk')?.text
  214. expect(sdk).not.toContain('denied:')
  215. expect(sdk).toContain('kept: {')
  216. runtime.behavior = request => Promise.resolve({
  217. logs: [],
  218. value: Object.keys(request.bindings[0]!.functions).sort().join(','),
  219. })
  220. const result = await runCode(ctx, 'return Object.keys(tools)', { agent })
  221. expect(result.isError).toBe(false)
  222. expect(result.content).toEqual([{ type: 'text', text: 'kept' }])
  223. })
  224. it.each(['code', 'both'] as const)('reserves run_code against scoped shadows and explicit restrictions in mode %s', async (mode) => {
  225. const { ctx, systemPrompt } = await setup({ mode })
  226. const { scope, agent } = await mintAgentScope(ctx)
  227. const impostor = defineContentToolFixture({
  228. name: RUN_CODE_NAME,
  229. description: 'Scoped impostor.',
  230. parameters: {},
  231. execute: () => Promise.resolve([{ type: 'text' as const, text: 'impostor' }]),
  232. })
  233. expect(() => scope.ctx.tools.register(impostor)).toThrow(/reserved for the Code Mode presentation transport/)
  234. expect(() => ctx.tools.register(impostor)).toThrow(/reserved for the Code Mode presentation transport/)
  235. expect(() => scope.ctx.tools.restrict({ allow: [RUN_CODE_NAME] })).toThrow(/cannot name reserved Code Mode presentation transport/)
  236. expect(() => scope.ctx.tools.restrict({ deny: [RUN_CODE_NAME] })).toThrow(/cannot name reserved Code Mode presentation transport/)
  237. scope.ctx.systemPrompt.section({ name: 'scoped-note', order: 149, text: 'safe note' })
  238. scope.ctx.tools.register(defineContentToolFixture({
  239. name: 'scoped_safe',
  240. description: 'Safe scoped tool.',
  241. parameters: {},
  242. execute: () => Promise.resolve([{ type: 'text' as const, text: 'safe' }]),
  243. }))
  244. const assembly = await systemPrompt.assemble({ scope: agent })
  245. const transports = assembly.tools.filter(tool => tool.name === RUN_CODE_NAME)
  246. expect(transports).toHaveLength(1)
  247. expect(transports[0]?.description).toContain('Execute a TypeScript program')
  248. expect(assembly.sections.find(section => section.name === 'scoped-note')?.text).toBe('safe note')
  249. expect(assembly.sections.find(section => section.name === 'tools:sdk')?.text).toContain('scoped_safe:')
  250. expect(ctx.tools.get(RUN_CODE_NAME, agent)).toBe(ctx.tools.get(RUN_CODE_NAME))
  251. const result = await runCode(ctx, 'return 1', { agent })
  252. expect(result.content).toEqual([{ type: 'text', text: '(run_code completed with no output)' }])
  253. })
  254. it.each(['code', 'both'] as const)('keeps run_code in the toolOrder universe without exposing it as a restriction target in mode %s', async (mode) => {
  255. const { ctx, systemPrompt } = await setup({
  256. mode,
  257. toolOrder: [RUN_CODE_NAME, '<unlisted-tools>'],
  258. })
  259. registerEcho(ctx)
  260. const { agent } = await mintAgentScope(ctx)
  261. const assembly = await systemPrompt.assemble({ scope: agent })
  262. expect(assembly.tools.map(tool => tool.name)).toEqual(mode === 'code'
  263. ? [RUN_CODE_NAME]
  264. : [RUN_CODE_NAME, 'echo'])
  265. })
  266. it("never exposes run_code to programs, even under mode 'both' (no recursive dispatch path)", async () => {
  267. const { ctx, runtime } = await setup({ mode: 'both' })
  268. registerEcho(ctx)
  269. runtime.behavior = (request) => {
  270. expect(request.bindings[0]!.errorClass).toEqual({
  271. name: 'ToolCallError',
  272. memberNameProperty: 'toolName',
  273. })
  274. const functions = request.bindings[0]!.functions
  275. return Promise.resolve({
  276. logs: [],
  277. value: JSON.stringify({
  278. names: Object.keys(functions).sort(),
  279. // Own-property AND prototype-chain reads both come back empty —
  280. // there is no handle a program could re-enter run_code through.
  281. runCode: String(functions[RUN_CODE_NAME]),
  282. }),
  283. })
  284. }
  285. const result = await runCode(ctx, 'program')
  286. expect(result.isError).toBe(false)
  287. expect(JSON.parse((result.content[0] as { text: string }).text)).toEqual({ names: ['echo'], runCode: 'undefined' })
  288. })
  289. it('renders byte-identical SDK text across consecutive assemblies of an unchanged tool set', async () => {
  290. const { ctx, systemPrompt } = await setup({ mode: 'code' })
  291. registerEcho(ctx)
  292. const first = await systemPrompt.assemble()
  293. const second = await systemPrompt.assemble()
  294. const text = (assembly: typeof first) => assembly.sections.find(section => section.name === 'tools:sdk')?.text
  295. expect(text(first)).toBe(text(second))
  296. })
  297. it('rejects every assembly when a non-native mode has no code runtime', async () => {
  298. const { systemPrompt } = await setup({ mode: 'code', runtime: false })
  299. await expect(systemPrompt.assemble()).rejects.toThrow(/requires a code runtime/)
  300. })
  301. it("rejects every assembly when the runtime's language is not typescript", async () => {
  302. const { systemPrompt } = await setup({ mode: 'code', runtime: { language: 'python' } })
  303. await expect(systemPrompt.assemble()).rejects.toThrow(/language is "python"/)
  304. })
  305. it("rejects the assembly when toolOrder names a native tool that mode 'code' no longer contributes", async () => {
  306. const { ctx, systemPrompt } = await setup({ mode: 'code', toolOrder: ['echo', '<unlisted-tools>'] })
  307. registerEcho(ctx)
  308. await expect(systemPrompt.assemble()).rejects.toThrow(/toolOrder lists unregistered tool "echo"/)
  309. })
  310. it('removes run_code and the SDK section when the registry fiber disposes (HMR safety)', async () => {
  311. const ctx = new Context()
  312. await ctx.plugin(SystemPrompt, {})
  313. await ctx.plugin(FakeRuntime, {})
  314. const fiber = await ctx.plugin(ToolRegistry, { mode: 'code' })
  315. expect(ctx.tools.get(RUN_CODE_NAME)).toBeDefined()
  316. await fiber.dispose()
  317. const assembly = await ctx.systemPrompt.assemble()
  318. expect(assembly.tools).toEqual([])
  319. expect(assembly.sections.some(section => section.name === 'tools:sdk')).toBe(false)
  320. })
  321. })
  322. describe('the sub-dispatch scheduler (native concurrency contract)', () => {
  323. /** Register a tool whose calls resolve only when the test releases them; returns live-call telemetry. */
  324. function registerGated(ctx: Context, name: string, concurrencySafe: boolean) {
  325. const gates: (() => void)[] = []
  326. let live = 0
  327. let peak = 0
  328. const order: string[] = []
  329. ctx.tools.register(defineTool({
  330. name,
  331. description: `Gated tool ${name}.`,
  332. parameters: { id: { type: 'string', required: true } },
  333. output: {
  334. schema: { type: 'string' },
  335. render: (_args, value) => [{ type: 'text', text: value }],
  336. },
  337. ...concurrencySafe ? { isConcurrencySafe: () => true } : {},
  338. async execute(args, exec) {
  339. order.push(`start:${args.id}`)
  340. live++
  341. peak = Math.max(peak, live)
  342. // Abort-observing like a real tool: the run-scoped abort releases the
  343. // gate so the bridge's drain reaches quiescence.
  344. await new Promise<void>((release) => {
  345. gates.push(release)
  346. exec.signal.addEventListener('abort', () => { release() }, { once: true })
  347. })
  348. live--
  349. order.push(`end:${args.id}`)
  350. return `${name}:${args.id}`
  351. },
  352. }))
  353. const release = (): void => { gates.shift()?.() }
  354. const releaseAll = (): void => { while (gates.length > 0) gates.shift()!() }
  355. return { order, release, releaseAll, peakLive: () => peak, pending: () => gates.length }
  356. }
  357. it('overlaps concurrency-safe calls under Promise.all and logs a start event per dispatch', async () => {
  358. const { ctx, runtime } = await setup({ mode: 'code' })
  359. const gated = registerGated(ctx, 'safe_read', true)
  360. const { agent, events } = fakeAgent()
  361. runtime.behavior = async (request) => {
  362. const tools = request.bindings[0]!.functions
  363. const all = Promise.all([
  364. tools.safe_read!({ id: 'a' }),
  365. tools.safe_read!({ id: 'b' }),
  366. tools.safe_read!({ id: 'c' }),
  367. ])
  368. // All three must be START-able without any completion (overlap proof).
  369. await expect.poll(() => gated.pending()).toBe(3)
  370. gated.releaseAll()
  371. return { logs: [], value: (await all).map(String).join(',') }
  372. }
  373. const result = await runCode(ctx, 'program', { agent })
  374. expect(result.isError).toBe(false)
  375. expect(gated.peakLive()).toBe(3)
  376. if (result.isError) throw new Error('expected success')
  377. expect(result.value).toMatchObject({ result: 'safe_read:a,safe_read:b,safe_read:c' })
  378. // One start per dispatch, paired with its settle by subCallId, starts in submission order.
  379. const starts = events.filter(event => event.type === 'tool/code-dispatch-start').map(event => event.data as { subCallId: string })
  380. const settles = events.filter(event => event.type === 'tool/code-dispatch').map(event => event.data as { subCallId: string })
  381. expect(starts.map(start => start.subCallId)).toEqual(['call-1:code:1', 'call-1:code:2', 'call-1:code:3'])
  382. expect(new Set(settles.map(settle => settle.subCallId))).toEqual(new Set(starts.map(start => start.subCallId)))
  383. })
  384. it('an exclusive call bars overlap: safe calls drain first, it runs alone, later calls wait', async () => {
  385. const { ctx, runtime } = await setup({ mode: 'code' })
  386. const safe = registerGated(ctx, 'safe_read', true)
  387. const unsafe = registerGated(ctx, 'writer', false)
  388. runtime.behavior = async (request) => {
  389. const tools = request.bindings[0]!.functions
  390. const reads = [tools.safe_read!({ id: 'r1' }), tools.safe_read!({ id: 'r2' })]
  391. const write = tools.writer!({ id: 'w' })
  392. const tail = tools.safe_read!({ id: 'r3' })
  393. await expect.poll(() => safe.pending()).toBe(2)
  394. // The exclusive call must NOT have started while the pool is live.
  395. expect(unsafe.pending()).toBe(0)
  396. safe.releaseAll()
  397. await expect.poll(() => unsafe.pending()).toBe(1)
  398. // The trailing safe call must NOT start while the exclusive one runs.
  399. expect(safe.pending()).toBe(0)
  400. unsafe.release()
  401. await expect.poll(() => safe.pending()).toBe(1)
  402. safe.releaseAll()
  403. await Promise.all([...reads, write, tail])
  404. return { logs: [], value: 'ordered' }
  405. }
  406. const result = await runCode(ctx, 'program')
  407. expect(result.isError).toBe(false)
  408. expect(safe.order.slice(0, 2)).toEqual(['start:r1', 'start:r2'])
  409. expect(unsafe.order).toEqual(['start:w', 'end:w'])
  410. // r3 started only after w ended.
  411. expect(safe.order.indexOf('start:r3')).toBeGreaterThan(safe.order.indexOf('end:r1'))
  412. })
  413. it('maxParallelSubCalls caps the overlap window', async () => {
  414. const { ctx, runtime } = await setup({ mode: 'code', maxParallelSubCalls: 2 })
  415. const gated = registerGated(ctx, 'safe_read', true)
  416. runtime.behavior = async (request) => {
  417. const tools = request.bindings[0]!.functions
  418. const all = Promise.all([
  419. tools.safe_read!({ id: 'a' }),
  420. tools.safe_read!({ id: 'b' }),
  421. tools.safe_read!({ id: 'c' }),
  422. ])
  423. await expect.poll(() => gated.pending()).toBe(2)
  424. // The third call waits for a slot.
  425. expect(gated.pending()).toBe(2)
  426. gated.release()
  427. await expect.poll(() => gated.pending()).toBe(2)
  428. gated.releaseAll()
  429. await all
  430. return { logs: [], value: 'capped' }
  431. }
  432. const result = await runCode(ctx, 'program')
  433. expect(result.isError).toBe(false)
  434. expect(gated.peakLive()).toBe(2)
  435. })
  436. it('a queued-unstarted call abandoned by run settlement logs no start event', async () => {
  437. const { ctx, runtime } = await setup({ mode: 'code' })
  438. const gated = registerGated(ctx, 'writer', false)
  439. const { agent, events } = fakeAgent()
  440. const abandoned: string[] = []
  441. runtime.behavior = async (request) => {
  442. const tools = request.bindings[0]!.functions
  443. // First exclusive call occupies the pool; the second queues unstarted.
  444. // Both rejections are captured (abandonment fires only at settlement,
  445. // AFTER this program has already failed — awaiting it here would deadlock).
  446. tools.writer!({ id: 'w1' }).catch(() => 'settled-under-abort')
  447. tools.writer!({ id: 'w2' }).catch((error: unknown) => {
  448. abandoned.push(error instanceof Error ? error.message : String(error))
  449. })
  450. await expect.poll(() => gated.pending()).toBe(1)
  451. // Fail the program while w1 is in flight and w2 is queued unstarted.
  452. throw new Error('program failed with a queued call')
  453. }
  454. const result = await runCode(ctx, 'program', { agent })
  455. expect(result.isError).toBe(true)
  456. const starts = events.filter(event => event.type === 'tool/code-dispatch-start').map(event => (event.data as { subCallId: string }).subCallId)
  457. const settles = events.filter(event => event.type === 'tool/code-dispatch').map(event => (event.data as { subCallId: string }).subCallId)
  458. // w1 started and settled under the abort; w2 never started and never
  459. // settled — no start event, no settle event, binding rejected with the
  460. // abandonment message at drain time.
  461. expect(starts).toEqual(['call-1:code:1'])
  462. expect(settles).toEqual(['call-1:code:1'])
  463. expect(abandoned).toEqual(['run_code run is over (run_code settled); writer tool call abandoned'])
  464. })
  465. })
  466. describe('the run_code dispatch bridge', () => {
  467. it('bridges tool calls, returns only the curated output, and logs one event per dispatch', async () => {
  468. const { ctx, runtime } = await setup({ mode: 'code' })
  469. const calls = registerEcho(ctx)
  470. const { agent, events } = fakeAgent()
  471. runtime.behavior = async (request) => {
  472. const tools = request.bindings[0]!.functions
  473. const first = await tools.echo!({ value: 'one' })
  474. const second = await tools.echo!({ value: 'two' })
  475. if (typeof first !== 'string' || typeof second !== 'string') throw new Error('echo returned a non-string')
  476. return { logs: [`saw ${first}`], value: second }
  477. }
  478. const result = await runCode(ctx, 'const …: string = …', { agent })
  479. expect(result.isError).toBe(false)
  480. if (result.isError) throw new Error('expected run_code success')
  481. expect(result.value).toEqual({ logs: ['saw echo:one'], result: 'echo:two' })
  482. expect(result.content).toEqual([{ type: 'text', text: 'saw echo:one\necho:two' }])
  483. expect(calls).toEqual([{ value: 'one' }, { value: 'two' }])
  484. const dispatches = events.filter(event => event.type === 'tool/code-dispatch')
  485. expect(dispatches.map(event => event.data)).toEqual([
  486. {
  487. parentCallId: 'call-1', subCallId: 'call-1:code:1', name: 'echo',
  488. arguments: { value: 'one' }, isError: false, content: [{ type: 'text', text: 'echo:one' }],
  489. },
  490. {
  491. parentCallId: 'call-1', subCallId: 'call-1:code:2', name: 'echo',
  492. arguments: { value: 'two' }, isError: false, content: [{ type: 'text', text: 'echo:two' }],
  493. },
  494. ])
  495. expect(result.meta).toBeUndefined()
  496. })
  497. it('exposes only an opaque parent token to nested result observers', async () => {
  498. const { ctx, runtime } = await setup({ mode: 'code' })
  499. registerEcho(ctx)
  500. runtime.behavior = async (request) => {
  501. await request.bindings[0]!.functions.echo!({ value: 'nested' })
  502. return { logs: [], value: 'done' }
  503. }
  504. // Freeze the nested observer's parent correlation. If that were the live
  505. // outer execution object, the timeout-style wrapper could not restore it.
  506. ctx.on('tools/execute', async (exec, next) => {
  507. if (exec.name !== RUN_CODE_NAME) return next()
  508. const previous = exec.signal
  509. exec.signal = new AbortController().signal
  510. const result = await next()
  511. exec.signal = previous
  512. return result
  513. })
  514. ctx.on('tools/result', (exec) => {
  515. if (exec.parent !== undefined) Object.freeze(exec.parent)
  516. })
  517. const result = await runCode(ctx, 'await tools.echo({ value: "nested" })')
  518. expect(result.isError).toBe(false)
  519. expect(result.content).toEqual([{ type: 'text', text: 'done' }])
  520. })
  521. it('serializes Promise.all dispatches: tool executions never overlap, in submission order', async () => {
  522. const { ctx, runtime } = await setup({ mode: 'code' })
  523. const intervals: [string, string][] = []
  524. let active = 0
  525. ctx.tools.register(defineTool({
  526. name: 'probe',
  527. description: 'Records execution overlap.',
  528. parameters: { id: { type: 'string', required: true } },
  529. output: {
  530. schema: { type: 'string' },
  531. render: (_args, value) => [{ type: 'text', text: value }],
  532. },
  533. async execute(args) {
  534. active++
  535. expect(active, 'probe executions overlapped').toBe(1)
  536. intervals.push(['enter', args.id])
  537. await new Promise(resolve => setTimeout(resolve, 20))
  538. intervals.push(['exit', args.id])
  539. active--
  540. return args.id
  541. },
  542. }))
  543. runtime.behavior = async (request) => {
  544. const tools = request.bindings[0]!.functions
  545. const values = await Promise.all([tools.probe!({ id: 'a' }), tools.probe!({ id: 'b' }), tools.probe!({ id: 'c' })])
  546. if (!values.every(value => typeof value === 'string')) throw new Error('probe returned a non-string')
  547. return { logs: [], value: values.join(',') }
  548. }
  549. const result = await runCode(ctx, 'program')
  550. expect(result.isError).toBe(false)
  551. expect(intervals).toEqual([
  552. ['enter', 'a'], ['exit', 'a'],
  553. ['enter', 'b'], ['exit', 'b'],
  554. ['enter', 'c'], ['exit', 'c'],
  555. ])
  556. expect(result.content[0]).toEqual({ type: 'text', text: 'a,b,c' })
  557. })
  558. it('rejects the program-side call when the tool errors, with the tool error text', async () => {
  559. const { ctx, runtime } = await setup({ mode: 'code' })
  560. ctx.tools.register(defineContentToolFixture({
  561. name: 'fail',
  562. description: 'Always fails.',
  563. parameters: {},
  564. execute(): Promise<never> { return Promise.reject(new Error('deliberate failure')) },
  565. }))
  566. runtime.behavior = async (request) => {
  567. try {
  568. await request.bindings[0]!.functions.fail!({})
  569. return { logs: [], value: 'unreachable' }
  570. } catch (error: unknown) {
  571. return { logs: [], value: `caught: ${error instanceof Error ? error.message : String(error)}` }
  572. }
  573. }
  574. const result = await runCode(ctx, 'program')
  575. expect(result.content[0]).toEqual({ type: 'text', text: 'caught: deliberate failure' })
  576. })
  577. it('a tools/pre-execute deny reaches the program as a binding rejection', async () => {
  578. const { ctx, runtime } = await setup({ mode: 'code' })
  579. registerEcho(ctx)
  580. ctx.on('tools/pre-execute', (exec, next) => {
  581. if (exec.name === 'echo') return Promise.resolve({ kind: 'deny' as const, reason: 'not on my watch' })
  582. return next()
  583. })
  584. runtime.behavior = async (request) => {
  585. try {
  586. await request.bindings[0]!.functions.echo!({ value: 'x' })
  587. return { logs: [], value: 'unreachable' }
  588. } catch (error: unknown) {
  589. return { logs: [], value: `denied: ${error instanceof Error ? error.message : String(error)}` }
  590. }
  591. }
  592. const result = await runCode(ctx, 'program')
  593. expect(result.content[0]?.type).toBe('text')
  594. expect((result.content[0] as { text: string }).text).toContain('not on my watch')
  595. })
  596. it('rejects a binding argument that is not lossless JSON, dispatching nothing', async () => {
  597. const { ctx, runtime } = await setup({ mode: 'code' })
  598. const calls = registerEcho(ctx)
  599. const { agent, events } = fakeAgent()
  600. runtime.behavior = async (request) => {
  601. try {
  602. await request.bindings[0]!.functions.echo!({ value: 'x', big: 1n })
  603. return { logs: [], value: 'unreachable' }
  604. } catch (error: unknown) {
  605. return { logs: [], value: error instanceof Error ? error.message : String(error) }
  606. }
  607. }
  608. const result = await runCode(ctx, 'program', { agent })
  609. expect((result.content[0] as { text: string }).text).toContain('lossless JSON')
  610. expect(calls).toEqual([])
  611. expect(events.filter(event => event.type === 'tool/code-dispatch')).toEqual([])
  612. })
  613. it('dispatches and logs independent snapshots of the same lossless JSON value', async () => {
  614. const { ctx, runtime } = await setup({ mode: 'code' })
  615. const calls = registerEcho(ctx)
  616. const { agent, events } = fakeAgent()
  617. runtime.behavior = async (request) => {
  618. const args = Object.assign(Object.create(null) as Record<string, unknown>, { value: 'x', nested: ['same'] })
  619. await request.bindings[0]!.functions.echo!(args)
  620. return { logs: [] }
  621. }
  622. await runCode(ctx, 'program', { agent })
  623. expect(calls).toEqual([{ value: 'x', nested: ['same'] }])
  624. const dispatch = events.find(event => event.type === 'tool/code-dispatch')?.data as SessionEventMap['tool/code-dispatch']
  625. expect(dispatch.arguments).toEqual({ value: 'x', nested: ['same'] })
  626. })
  627. it('defers sub-call additionalContexts onto the outer run_code result', async () => {
  628. const { ctx, runtime } = await setup({ mode: 'code' })
  629. registerEcho(ctx)
  630. ctx.on('tools/post-execute', (exec, _result, next): Promise<PostToolDecision> => {
  631. if (exec.name === 'echo') {
  632. return Promise.resolve({
  633. kind: 'accept' as const,
  634. additionalContexts: [{
  635. content: [{ type: 'text' as const, text: `context for ${exec.callId}` }],
  636. source: { kind: 'plugin' as const, plugin: 'test' },
  637. meta: { callId: exec.callId },
  638. }],
  639. })
  640. }
  641. return next()
  642. })
  643. runtime.behavior = async (request) => {
  644. await request.bindings[0]!.functions.echo!({ value: 'x' })
  645. await request.bindings[0]!.functions.echo!({ value: 'y' })
  646. return { logs: [], value: 'done' }
  647. }
  648. const result = await runCode(ctx, 'program')
  649. expect(result.isError).toBe(false)
  650. expect(result.additionalContexts).toEqual([
  651. {
  652. content: [{ type: 'text', text: 'context for call-1:code:1' }],
  653. source: { kind: 'plugin', plugin: 'test' },
  654. meta: { callId: 'call-1:code:1' },
  655. },
  656. {
  657. content: [{ type: 'text', text: 'context for call-1:code:2' }],
  658. source: { kind: 'plugin', plugin: 'test' },
  659. meta: { callId: 'call-1:code:2' },
  660. },
  661. ])
  662. })
  663. it('keeps sub-call contexts when run_code fails after the nested dispatch', async () => {
  664. const { ctx, runtime } = await setup({ mode: 'both' })
  665. registerEcho(ctx)
  666. ctx.on('tools/post-execute', (exec, _result, next): Promise<PostToolDecision> => {
  667. if (exec.name !== 'echo') return next()
  668. return Promise.resolve({
  669. kind: 'accept',
  670. additionalContexts: [{
  671. content: [{ type: 'text', text: 'nested context' }],
  672. source: { kind: 'plugin', plugin: 'test' },
  673. }],
  674. })
  675. })
  676. runtime.behavior = async (request) => {
  677. await request.bindings[0]!.functions.echo!({ value: 'x' })
  678. return { logs: [], error: { kind: 'exception', message: 'program failed later' } }
  679. }
  680. const result = await runCode(ctx, 'program')
  681. expect(result.isError).toBe(true)
  682. expect(result.additionalContexts).toEqual([{
  683. content: [{ type: 'text', text: 'nested context' }],
  684. source: { kind: 'plugin', plugin: 'test' },
  685. }])
  686. })
  687. it('converts a failed run into a structured isError result carrying kind, message, and captured logs', async () => {
  688. const { ctx, runtime } = await setup({ mode: 'code' })
  689. runtime.behavior = () => Promise.resolve({
  690. logs: ['got this far'],
  691. error: { kind: 'timeout', message: 'compute budget exhausted (300ms busy)' },
  692. })
  693. const result = await runCode(ctx, 'program')
  694. expect(result.isError).toBe(true)
  695. expect(result.error).toMatchObject({ info: { name: 'CodeRunFailedError', code: 'CODE_RUN_FAILED' } })
  696. const text = (result.content[0] as { text: string }).text
  697. expect(text).toContain('code run failed (timeout)')
  698. expect(text).toContain('compute budget exhausted')
  699. expect(text).toContain('got this far')
  700. })
  701. it('CodeRunFailedError is a HarnessError with the CODE_RUN_FAILED code', () => {
  702. const error = new CodeRunFailedError('boom')
  703. expect(error.code).toBe('CODE_RUN_FAILED')
  704. expect(error.name).toBe('CodeRunFailedError')
  705. })
  706. it('aborting the outer signal aborts the in-flight sub-dispatch and abandons queued ones', async () => {
  707. const { ctx, runtime } = await setup({ mode: 'code' })
  708. const seen: string[] = []
  709. let sawAbort = false
  710. ctx.tools.register(defineContentToolFixture({
  711. name: 'slow',
  712. description: 'Slow tool observing its signal.',
  713. parameters: { id: { type: 'string', required: true } },
  714. async execute(args, exec) {
  715. seen.push(args.id)
  716. await new Promise<void>((resolve) => {
  717. const timer = setTimeout(resolve, 500)
  718. exec.signal.addEventListener('abort', () => { sawAbort = true; clearTimeout(timer); resolve() }, { once: true })
  719. })
  720. return [{ type: 'text' as const, text: args.id }]
  721. },
  722. }))
  723. const controller = new AbortController()
  724. runtime.behavior = async (request) => {
  725. const tools = request.bindings[0]!.functions
  726. const calls = [tools.slow!({ id: 'first' }).catch(() => 'rejected'), tools.slow!({ id: 'second' }).catch(() => 'rejected')]
  727. setTimeout(() => { controller.abort('user-cancel') }, 50)
  728. await Promise.all(calls)
  729. // A real runtime would be terminated by the abort; the fake honors the
  730. // contract by reporting the abort as the run failure.
  731. return { logs: [], error: { kind: 'abort', message: 'user-cancel' } }
  732. }
  733. const result = await runCode(ctx, 'program', { signal: controller.signal })
  734. expect(result.isError).toBe(true)
  735. expect((result.content[0] as { text: string }).text).toContain('code run failed (abort)')
  736. expect(seen).toEqual(['first'])
  737. expect(sawAbort).toBe(true)
  738. })
  739. it('a runtime that starts a binding call and then REJECTS still reaches quiescence before returning', async () => {
  740. const { ctx, runtime } = await setup({ mode: 'code' })
  741. const { agent, events } = fakeAgent()
  742. let sawAbort = false
  743. let started!: () => void
  744. const inFlight = new Promise<void>((resolve) => { started = resolve })
  745. ctx.tools.register(defineContentToolFixture({
  746. name: 'slow',
  747. description: 'Slow tool observing its signal.',
  748. parameters: { id: { type: 'string', required: true } },
  749. async execute(args, exec) {
  750. started()
  751. await new Promise<void>((resolve) => {
  752. const timer = setTimeout(resolve, 500)
  753. exec.signal.addEventListener('abort', () => { sawAbort = true; clearTimeout(timer); resolve() }, { once: true })
  754. })
  755. return [{ type: 'text' as const, text: args.id }]
  756. },
  757. }))
  758. runtime.behavior = async (request) => {
  759. // Start a sub-dispatch, keep its rejection held, and fail the run once the tool is
  760. // genuinely in flight — a seam error after work has begun.
  761. request.bindings[0]!.functions.slow!({ id: 'orphan' }).catch(() => 'held')
  762. await inFlight
  763. throw new Error('backend exploded')
  764. }
  765. const result = await runCode(ctx, 'program', { agent })
  766. expect(result.isError).toBe(true)
  767. expect((result.content[0] as { text: string }).text).toContain('backend exploded')
  768. // Quiescence held: the in-flight sub-dispatch was aborted and its event
  769. // logged INSIDE the run_code execution, not after it returned.
  770. expect(sawAbort).toBe(true)
  771. expect(events.filter(event => event.type === 'tool/code-dispatch').map(event => (event.data as { name: string }).name)).toEqual(['slow'])
  772. })
  773. it('runs without an owning agent: dispatches work, event logging is skipped', async () => {
  774. const { ctx, runtime } = await setup({ mode: 'code' })
  775. const calls = registerEcho(ctx)
  776. runtime.behavior = async (request) => {
  777. await request.bindings[0]!.functions.echo!({ value: 'x' })
  778. return { logs: [], value: 'ok' }
  779. }
  780. const result = await runCode(ctx, 'program')
  781. expect(result.isError).toBe(false)
  782. expect(calls).toEqual([{ value: 'x' }])
  783. })
  784. it('executing run_code under a missing runtime is a structured isError, not a crash', async () => {
  785. const ctx = new Context()
  786. await ctx.plugin(SystemPrompt, {})
  787. await ctx.plugin(ToolRegistry, { mode: 'code' })
  788. const result = await runCode(ctx, 'program')
  789. expect(result.isError).toBe(true)
  790. expect((result.content[0] as { text: string }).text).toContain('requires a code runtime')
  791. })
  792. it('presents the model-authored description as the execute-card title over the program input', async () => {
  793. const { ctx } = await setup({ mode: 'code' })
  794. const tool = ctx.tools.get(RUN_CODE_NAME)!
  795. // The description labels the card (the bash description precedent); the
  796. // program itself remains the expanded raw input.
  797. expect(tool.presentCall?.({ code: 'return 1', description: 'Return the constant one' })).toEqual({
  798. card: 'generic',
  799. title: 'Return the constant one',
  800. kind: 'execute',
  801. rawInput: 'return 1',
  802. })
  803. })
  804. it('rejects a whitespace-only description with a structured isError', async () => {
  805. const { ctx } = await setup({ mode: 'code' })
  806. const result = await runCode(ctx, 'return 1', { description: ' ' })
  807. expect(result.isError).toBe(true)
  808. expect((result.content[0] as { text: string }).text).toContain('invalid description')
  809. })
  810. it.each([
  811. ['logs only', { logs: ['printed'] }, 'printed'],
  812. ['result only', { logs: [], value: 'returned' }, 'returned'],
  813. ['logs plus result', { logs: ['printed'], value: 'returned' }, 'printed\nreturned'],
  814. ['no output', { logs: [] }, '(run_code completed with no output)'],
  815. ] as [string, CodeRunResult, string][])('keeps %s in durable content without a result presenter', async (_name, output, text) => {
  816. const { ctx, runtime } = await setup({ mode: 'code' })
  817. runtime.behavior = () => Promise.resolve(output)
  818. const result = await runCode(ctx, 'return 1')
  819. const tool = ctx.tools.get(RUN_CODE_NAME)!
  820. expect(result.content).toEqual([{ type: 'text', text }])
  821. // Surfaces keep the pending program title and render this durable content
  822. // through their generic fallback. Omitting a result view also prevents the
  823. // host frame from carrying the same raw content a second time.
  824. expect('presentResult' in tool).toBe(false)
  825. })
  826. it('keeps a post-policy spill preview in durable content without a result presenter', async () => {
  827. const { ctx, runtime } = await setup({ mode: 'code' })
  828. const preview = 'HEAD\n\n(Omitted 100 bytes. Full formatted result stored at: /tmp/run-code.txt.)\n\nTAIL'
  829. runtime.behavior = () => Promise.resolve({ logs: ['printed'], value: 'returned' })
  830. ctx.on('tools/post-execute', (exec, _result, next): Promise<PostToolDecision> => {
  831. if (exec.name !== RUN_CODE_NAME) return next()
  832. return Promise.resolve({ kind: 'accept', content: [{ type: 'text', text: preview }] })
  833. })
  834. const result = await runCode(ctx, 'return 1')
  835. const tool = ctx.tools.get(RUN_CODE_NAME)!
  836. expect(result.content).toEqual([{ type: 'text', text: preview }])
  837. expect('presentResult' in tool).toBe(false)
  838. })
  839. it('keeps canonical failure content durable without a result presenter', async () => {
  840. const { ctx, runtime } = await setup({ mode: 'code' })
  841. runtime.behavior = () => Promise.resolve({
  842. logs: ['captured before failure'],
  843. error: { kind: 'output-limit', message: 'outer output exceeded 8 bytes' },
  844. })
  845. const result = await runCode(ctx, 'return 1')
  846. const tool = ctx.tools.get(RUN_CODE_NAME)!
  847. expect(result.isError).toBe(true)
  848. expect(result.content).toEqual([{
  849. type: 'text',
  850. text: 'Error: code run failed (output-limit): outer output exceeded 8 bytes\nCaptured output:\ncaptured before failure',
  851. }])
  852. expect('presentResult' in tool).toBe(false)
  853. })
  854. it('logs the complete sub-result content verbatim, non-text blocks and long text included', async () => {
  855. const { ctx, runtime } = await setup({ mode: 'code' })
  856. const { agent, events } = fakeAgent()
  857. const long = 'x'.repeat(300)
  858. ctx.tools.register(defineTool({
  859. name: 'mixed',
  860. description: 'Returns mixed content.',
  861. parameters: {},
  862. output: {
  863. schema: { type: 'string' },
  864. render: () => [
  865. { type: 'text', text: long },
  866. { type: 'reasoning', text: 'hidden' },
  867. ],
  868. },
  869. execute() {
  870. return Promise.resolve('mixed-value')
  871. },
  872. }))
  873. runtime.behavior = async (request) => {
  874. const value = await request.bindings[0]!.functions.mixed!({})
  875. return { logs: [], value }
  876. }
  877. const result = await runCode(ctx, 'program', { agent })
  878. expect(result.isError).toBe(false)
  879. expect((result.content[0] as { text: string }).text).toBe('mixed-value')
  880. const dispatch = events.find(event => event.type === 'tool/code-dispatch')?.data as SessionEventMap['tool/code-dispatch']
  881. expect(dispatch.content).toEqual([
  882. { type: 'text', text: long },
  883. { type: 'reasoning', text: 'hidden' },
  884. ])
  885. })
  886. it('rejects undefined, getter-throwing, exotic, and unrepresentable binding arguments before dispatch', async () => {
  887. const { ctx, runtime } = await setup({ mode: 'code' })
  888. const calls = registerEcho(ctx)
  889. const { agent, events } = fakeAgent()
  890. runtime.behavior = async (request) => {
  891. const echo = request.bindings[0]!.functions.echo!
  892. const catchMessage = (promise: Promise<unknown>) => promise.then(() => 'resolved', (error: unknown) => error instanceof Error ? error.message : String(error))
  893. return {
  894. logs: [],
  895. value: [
  896. // Root undefined must reject up front: the event log rejects it as
  897. // data, and nothing may execute unlogged.
  898. await catchMessage(echo(undefined)),
  899. await catchMessage(echo(Object.defineProperty({}, 'bad', { enumerable: true, get() { throw 'raw-throw' } }))),
  900. await catchMessage(echo(Object.defineProperty({}, 'bad', { enumerable: true, get() { throw new Error('error-throw') } }))),
  901. await catchMessage(echo(new Date(0))),
  902. // A bare function is a value JSON cannot represent at all.
  903. await catchMessage(echo(() => 1)),
  904. ].join(' | '),
  905. }
  906. }
  907. const result = await runCode(ctx, 'program', { agent })
  908. const text = (result.content[0] as { text: string }).text
  909. expect(text).toContain('call the tool with an arguments object')
  910. expect(text).toContain('lossless JSON: raw-throw')
  911. expect(text).toContain('lossless JSON: error-throw')
  912. expect(text.match(/tool arguments must be lossless JSON/g)).toHaveLength(5)
  913. // None dispatched or logged.
  914. expect(calls).toEqual([])
  915. expect(events.filter(event => event.type === 'tool/code-dispatch')).toEqual([])
  916. })
  917. it('dispatches and durably logs binding arguments deeper than the structured-clone call stack', async () => {
  918. const { ctx, runtime } = await setup({ mode: 'code' })
  919. const depth = 5_000
  920. let observedDepth = 0
  921. let observedLeaf: JsonValue | undefined
  922. ctx.tools.register(defineTool({
  923. name: 'deep_args',
  924. description: 'Measure a deeply nested JSON argument.',
  925. parameters: { nested: { type: 'json', required: true } },
  926. output: {
  927. schema: { type: 'integer' },
  928. render: (_args, value) => [{ type: 'text', text: String(value) }],
  929. },
  930. execute(args) {
  931. let cursor = args.nested
  932. while (Array.isArray(cursor)) {
  933. if (cursor.length !== 1) throw new Error('expected one item per nesting layer')
  934. observedDepth++
  935. cursor = cursor[0]!
  936. }
  937. observedLeaf = cursor
  938. return Promise.resolve(observedDepth)
  939. },
  940. }))
  941. const session = new Session(SessionId('deep-code-arguments'))
  942. const agent = { session } as Agent
  943. runtime.behavior = async (request) => {
  944. let nested: JsonValue = 'leaf'
  945. for (let index = 0; index < depth; index++) nested = [nested]
  946. const value = await request.bindings[0]!.functions.deep_args!({ nested })
  947. return { logs: [], value }
  948. }
  949. const result = await runCode(ctx, 'return tools.deep_args(...)', { agent })
  950. expect(result.isError).toBe(false)
  951. expect(result.isError ? undefined : result.value).toEqual({ logs: [], result: depth })
  952. expect({ observedDepth, observedLeaf }).toEqual({ observedDepth: depth, observedLeaf: 'leaf' })
  953. const dispatch = session.events.find(event => event.type === 'tool/code-dispatch')
  954. if (dispatch === undefined) throw new Error('expected a durable tool/code-dispatch event')
  955. const logged = dispatch.data.arguments as { nested: JsonValue }
  956. let loggedDepth = 0
  957. let loggedCursor = logged.nested
  958. while (Array.isArray(loggedCursor)) {
  959. if (loggedCursor.length !== 1) throw new Error('expected one logged item per nesting layer')
  960. loggedDepth++
  961. loggedCursor = loggedCursor[0]!
  962. }
  963. expect({ loggedDepth, loggedCursor }).toEqual({ loggedDepth: depth, loggedCursor: 'leaf' })
  964. })
  965. it('gives the tool and durable log the same immutable argument value', async () => {
  966. const { ctx, runtime } = await setup({ mode: 'code' })
  967. const { agent, events } = fakeAgent()
  968. let mutationSucceeded: boolean | undefined
  969. ctx.tools.register(defineContentToolFixture({
  970. name: 'mutator',
  971. description: 'Attempts to mutate its args object.',
  972. parameters: { list: { type: 'array', required: true } },
  973. execute(args) {
  974. mutationSucceeded = Reflect.set(args.list, 1, 'injected-by-tool')
  975. return Promise.resolve([{ type: 'text' as const, text: 'protected' }])
  976. },
  977. }))
  978. runtime.behavior = async (request) => {
  979. await request.bindings[0]!.functions.mutator!({ list: ['original'] })
  980. return { logs: [] }
  981. }
  982. const result = await runCode(ctx, 'program', { agent })
  983. expect(result.isError).toBe(false)
  984. expect(mutationSucceeded).toBe(false)
  985. const dispatch = events.find(event => event.type === 'tool/code-dispatch')?.data as SessionEventMap['tool/code-dispatch']
  986. expect(dispatch.arguments).toEqual({ list: ['original'] })
  987. })
  988. it('exposes a tool named __proto__ as an ordinary own binding', async () => {
  989. const { ctx, runtime } = await setup({ mode: 'code' })
  990. ctx.tools.register(defineTool({
  991. name: '__proto__',
  992. description: 'A prototype-colliding tool name.',
  993. parameters: {},
  994. output: {
  995. schema: { type: 'string' },
  996. render: (_args, value) => [{ type: 'text', text: value }],
  997. },
  998. execute() { return Promise.resolve('proto-tool-ok') },
  999. }))
  1000. runtime.behavior = async (request) => {
  1001. const functions = request.bindings[0]!.functions
  1002. expect(Object.getPrototypeOf(functions)).toBeNull()
  1003. const value = await functions['__proto__']!({})
  1004. return { logs: [], value }
  1005. }
  1006. const result = await runCode(ctx, 'program')
  1007. expect(result.isError).toBe(false)
  1008. expect(result.content[0]).toEqual({ type: 'text', text: 'proto-tool-ok' })
  1009. })
  1010. it('renders every non-string JSON root as pretty JSON while preserving strings raw', async () => {
  1011. const { ctx, runtime } = await setup({ mode: 'code' })
  1012. runtime.behavior = () => Promise.resolve({ logs: [], value: { n: 42, ok: true } })
  1013. expect((await runCode(ctx, 'object')).content[0]).toEqual({ type: 'text', text: '{\n "n": 42,\n "ok": true\n}' })
  1014. runtime.behavior = () => Promise.resolve({ logs: [], value: {} })
  1015. expect((await runCode(ctx, 'empty object')).content[0]).toEqual({ type: 'text', text: '{}' })
  1016. const nested = { outer: [{ inner: true }] }
  1017. runtime.behavior = () => Promise.resolve({ logs: [], value: nested })
  1018. expect((await runCode(ctx, 'nested')).content[0]).toEqual({ type: 'text', text: JSON.stringify(nested, null, 2) })
  1019. runtime.behavior = () => Promise.resolve({ logs: [], value: ['x', 2] })
  1020. expect((await runCode(ctx, 'array')).content[0]).toEqual({ type: 'text', text: '[\n "x",\n 2\n]' })
  1021. runtime.behavior = () => Promise.resolve({ logs: [], value: [] })
  1022. expect((await runCode(ctx, 'empty array')).content[0]).toEqual({ type: 'text', text: '[]' })
  1023. runtime.behavior = () => Promise.resolve({ logs: [], value: null })
  1024. expect((await runCode(ctx, 'null')).content[0]).toEqual({ type: 'text', text: 'null' })
  1025. runtime.behavior = () => Promise.resolve({ logs: [], value: 'raw' })
  1026. expect((await runCode(ctx, 'string')).content[0]).toEqual({ type: 'text', text: 'raw' })
  1027. runtime.behavior = () => Promise.resolve({ logs: [] })
  1028. const absent = await runCode(ctx, 'undefined')
  1029. expect(absent.content[0]).toEqual({ type: 'text', text: '(run_code completed with no output)' })
  1030. expect(absent.isError ? undefined : absent.value).toEqual({ logs: [] })
  1031. })
  1032. it('renders deeply nested JSON without recursive traversal or quadratic indentation', async () => {
  1033. const { ctx, runtime } = await setup({ mode: 'code' })
  1034. let value: JsonValue = {
  1035. emptyArray: [],
  1036. emptyObject: {},
  1037. pair: ['leaf', 2],
  1038. record: { first: true, second: null },
  1039. }
  1040. for (let depth = 0; depth < 5_000; depth++) value = [value]
  1041. runtime.behavior = () => Promise.resolve({ logs: [], value })
  1042. const result = await runCode(ctx, 'deep result')
  1043. expect(result.isError).toBe(false)
  1044. const text = (result.content[0] as { type: 'text'; text: string }).text
  1045. expect(text.startsWith('[\n [\n [')).toBe(true)
  1046. expect(text).toContain('"leaf"')
  1047. expect(text.endsWith(']')).toBe(true)
  1048. expect(text.length).toBeLessThan(11_000)
  1049. })
  1050. it('short-circuits a pre-aborted outer signal before the code runtime', async () => {
  1051. const { ctx, runtime } = await setup({ mode: 'code' })
  1052. const calls = registerEcho(ctx)
  1053. runtime.behavior = (request) => {
  1054. // The fake honors the seam contract for an already-aborted signal.
  1055. if (request.signal?.aborted) return Promise.resolve({ logs: [], error: { kind: 'abort' as const, message: String(request.signal.reason) } })
  1056. return Promise.resolve({ logs: [], value: 'unreachable' })
  1057. }
  1058. const controller = new AbortController()
  1059. controller.abort('too-late')
  1060. const result = await runCode(ctx, 'program', { signal: controller.signal })
  1061. expect(result.isError).toBe(true)
  1062. expect(result).toEqual({
  1063. content: [{ type: 'text', text: 'Error: tool call aborted before dispatch' }],
  1064. isError: true,
  1065. error: {
  1066. message: 'tool call aborted before dispatch',
  1067. info: { name: 'AbortError', code: TOOL_ABORTED_BEFORE_DISPATCH },
  1068. },
  1069. })
  1070. expect(runtime.lastRequest).toBeUndefined()
  1071. expect(calls).toEqual([])
  1072. })
  1073. it('reports cancellation after rejecting a late binding without dispatching it', async () => {
  1074. const { ctx, runtime } = await setup({ mode: 'code' })
  1075. const calls = registerEcho(ctx)
  1076. const controller = new AbortController()
  1077. runtime.behavior = async (request) => {
  1078. controller.abort('cancelled-mid-run')
  1079. const message = await request.bindings[0]!.functions.echo!({ value: 'x' })
  1080. .then(() => 'resolved', (error: unknown) => error instanceof Error ? error.message : String(error))
  1081. return { logs: [], value: message }
  1082. }
  1083. const result = await runCode(ctx, 'program', { signal: controller.signal })
  1084. expect(result.isError).toBe(true)
  1085. expect(result.error).toEqual({
  1086. message: 'tool call aborted',
  1087. info: { name: 'AbortError', code: 'ABORTED' },
  1088. })
  1089. expect((result.content[0] as { text: string }).text).toBe('Error: tool call aborted')
  1090. expect(calls).toEqual([])
  1091. })
  1092. it('a tool/code-dispatch event never derives a model message', () => {
  1093. const session = new Session(SessionId('code-mode-derive'))
  1094. session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }, { surfaceOp: 'append' })
  1095. session.append('tool/code-dispatch', {
  1096. parentCallId: CallId('p1'),
  1097. subCallId: CallId('p1:code:1'),
  1098. name: 'echo',
  1099. arguments: { value: 'x' },
  1100. isError: false,
  1101. content: [{ type: 'text', text: 'echo:x' }],
  1102. })
  1103. const derived = session.deriveMessages()
  1104. expect(derived).toHaveLength(1)
  1105. expect(derived[0]?.role).toBe('user')
  1106. })
  1107. it('defaults to native mode under direct construction with no config', async () => {
  1108. const ctx = new Context()
  1109. await ctx.plugin(SystemPrompt, {})
  1110. const registry = new ToolRegistry(ctx)
  1111. expect(registry.get(RUN_CODE_NAME)).toBeUndefined()
  1112. const assembly = await ctx.systemPrompt.assemble()
  1113. expect(assembly.sections.some(section => section.name === 'tools:sdk')).toBe(false)
  1114. })
  1115. })