code-mode.spec.ts 38 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822
  1. import { describe, expect, it } from 'vitest'
  2. import { Context } from 'cordis'
  3. import { CallId } from '@deepseek-ai/dsh-llm'
  4. import { createScope } from '@deepseek-ai/dsh-scope'
  5. import type { Scope } from '@deepseek-ai/dsh-scope'
  6. import SystemPrompt from '@deepseek-ai/dsh-system-prompt'
  7. import { CodeRuntime } from '@deepseek-ai/dsh-code-runtime'
  8. import type { CodeRunRequest, CodeRunResult } from '@deepseek-ai/dsh-code-runtime'
  9. import ToolRegistry, { CodeRunFailedError, RUN_CODE_NAME, defineTool } from '@deepseek-ai/dsh-tools'
  10. import type { Config, PostToolDecision, ToolExecutionResult } from '@deepseek-ai/dsh-tools'
  11. import { AgentId } from '@deepseek-ai/dsh-agent'
  12. import type { Agent } from '@deepseek-ai/dsh-agent'
  13. import { Session, SessionId } from '@deepseek-ai/dsh-session'
  14. import type { SessionEventMap } from '@deepseek-ai/dsh-session'
  15. /**
  16. * Code Mode unit tier (per the RFC's plan): provider contribution per mode,
  17. * misconfiguration rejections, the run_code dispatch bridge (serialization,
  18. * abort, JSON normalization, error mapping, events, quiescence), and HMR
  19. * safety — all against an in-repo fake runtime, exactly the
  20. * interface/implementation/consumer shape the seam promises.
  21. */
  22. /** A scriptable in-repo CodeRuntime: each test sets `behavior` to drive the bindings however it needs. */
  23. class FakeRuntime extends CodeRuntime {
  24. readonly language: string
  25. readonly isolation = 'fake'
  26. behavior: (request: CodeRunRequest) => Promise<CodeRunResult> = () => Promise.resolve({ logs: [] })
  27. lastRequest?: CodeRunRequest
  28. constructor(ctx: Context, config: { language?: string } = {}) {
  29. super(ctx)
  30. this.language = config.language ?? 'typescript'
  31. }
  32. run(request: CodeRunRequest): Promise<CodeRunResult> {
  33. this.lastRequest = request
  34. return this.behavior(request)
  35. }
  36. }
  37. interface SetupOptions {
  38. mode?: Config['mode']
  39. runtime?: false | { language?: string }
  40. toolOrder?: string[]
  41. }
  42. async function setup(options: SetupOptions = {}) {
  43. const ctx = new Context()
  44. await ctx.plugin(SystemPrompt, { ...options.toolOrder ? { toolOrder: options.toolOrder } : {} })
  45. await ctx.plugin(ToolRegistry, { mode: options.mode ?? 'code' })
  46. let runtime: FakeRuntime | undefined
  47. if (options.runtime !== false) {
  48. await ctx.plugin(FakeRuntime, options.runtime ?? {})
  49. runtime = ctx.codeRuntime as FakeRuntime
  50. }
  51. return { ctx, tools: ctx.tools, systemPrompt: ctx.systemPrompt, runtime: runtime! }
  52. }
  53. /** Mint one production-shaped agent scope that can register scoped tool policy. */
  54. async function mintAgentScope(ctx: Context, name = 'scoped'): Promise<{ scope: Scope; agent: Agent }> {
  55. const agent = { id: AgentId(name) } as Agent
  56. let scope!: Scope
  57. await ctx.plugin(Object.assign((inner: Context) => { scope = createScope(inner, agent) },
  58. { inject: ['tools', 'systemPrompt'] }))
  59. return { scope, agent }
  60. }
  61. /** Register a trivial echo tool; returns the calls it received. */
  62. function registerEcho(ctx: Context, name = 'echo'): unknown[] {
  63. const calls: unknown[] = []
  64. ctx.tools.register(defineTool({
  65. name,
  66. description: `Echo tool ${name}.`,
  67. parameters: { value: { type: 'string', required: true } },
  68. execute(args) {
  69. calls.push(args)
  70. return Promise.resolve([{ type: 'text' as const, text: `${name}:${args.value}` }])
  71. },
  72. }))
  73. return calls
  74. }
  75. /** A structural fake of the owning agent: captures session appends. */
  76. function fakeAgent(): { agent: Agent; events: { type: string; data: unknown }[] } {
  77. const events: { type: string; data: unknown }[] = []
  78. const agent = {
  79. session: {
  80. append: (type: string, data: unknown) => { events.push({ type, data }) },
  81. },
  82. } as unknown as Agent
  83. return { agent, events }
  84. }
  85. /** Dispatch run_code through the registry pipeline, as the loop would. */
  86. async function runCode(ctx: Context, code: string, extras: { agent?: Agent; signal?: AbortSignal } = {}): Promise<ToolExecutionResult> {
  87. return ctx.tools.execute({
  88. callId: CallId('call-1'),
  89. name: RUN_CODE_NAME,
  90. arguments: { code },
  91. ...extras.agent ? { agent: extras.agent } : {},
  92. ...extras.signal ? { signal: extras.signal } : {},
  93. })
  94. }
  95. describe('mode-aware wire contribution', () => {
  96. it("mode 'native' contributes every schema, no run_code, no SDK section — and needs no runtime", async () => {
  97. const { ctx, systemPrompt } = await setup({ mode: 'native', runtime: false })
  98. registerEcho(ctx)
  99. const assembly = await systemPrompt.assemble()
  100. expect(assembly.tools.map(tool => tool.name)).toEqual(['echo'])
  101. expect(assembly.sections.some(section => section.name === 'tools:sdk')).toBe(false)
  102. })
  103. it("mode 'code' contributes exactly [run_code] plus the SDK section declaring the other tools", async () => {
  104. const { ctx, systemPrompt } = await setup({ mode: 'code' })
  105. registerEcho(ctx)
  106. const assembly = await systemPrompt.assemble()
  107. expect(assembly.tools.map(tool => tool.name)).toEqual([RUN_CODE_NAME])
  108. const sdk = assembly.sections.find(section => section.name === 'tools:sdk')
  109. expect(sdk?.text).toContain('declare const tools: {')
  110. expect(sdk?.text).toContain('echo(args:')
  111. expect(sdk?.text).not.toContain('run_code(args:')
  112. })
  113. it.each(['code', 'both'] as const)('restores Code Mode infrastructure after assembly listeners in mode %s', async (mode) => {
  114. const { ctx, systemPrompt } = await setup({ mode })
  115. registerEcho(ctx)
  116. ctx.on('system-prompt/assemble', async (_assembly, _context, next) => {
  117. const assembly = await next()
  118. return {
  119. ...assembly,
  120. sections: assembly.sections.filter(section => section.name !== 'tools:sdk'),
  121. tools: assembly.tools.filter(tool => tool.name !== RUN_CODE_NAME),
  122. }
  123. }, { prepend: true })
  124. const assembly = await systemPrompt.assemble()
  125. expect(assembly.sections.some(section => section.name === 'tools:sdk')).toBe(true)
  126. expect(assembly.tools.some(tool => tool.name === RUN_CODE_NAME)).toBe(true)
  127. })
  128. it("mode 'both' contributes every native schema plus run_code, and the SDK section", async () => {
  129. const { ctx, systemPrompt } = await setup({ mode: 'both' })
  130. registerEcho(ctx)
  131. const assembly = await systemPrompt.assemble()
  132. expect(assembly.tools.map(tool => tool.name)).toEqual(['echo', RUN_CODE_NAME])
  133. expect(assembly.sections.some(section => section.name === 'tools:sdk')).toBe(true)
  134. })
  135. it.each(['code', 'both'] as const)('keeps the run_code transport outside scoped allow-list filtering in mode %s', async (mode) => {
  136. const { ctx, systemPrompt, runtime } = await setup({ mode })
  137. registerEcho(ctx, 'echo')
  138. registerEcho(ctx, 'hidden')
  139. const { scope, agent } = await mintAgentScope(ctx)
  140. const lift = scope.ctx.tools.restrict({ allow: ['echo'] })
  141. const assembly = await systemPrompt.assemble({ scope: agent })
  142. expect(assembly.tools.map(tool => tool.name)).toEqual(mode === 'code'
  143. ? [RUN_CODE_NAME]
  144. : ['echo', RUN_CODE_NAME])
  145. const sdk = assembly.sections.find(section => section.name === 'tools:sdk')?.text
  146. expect(sdk).toContain('echo(args:')
  147. expect(sdk).not.toContain('hidden(args:')
  148. runtime.behavior = request => Promise.resolve({
  149. logs: [],
  150. value: Object.keys(request.bindings[0]!.functions).sort().join(','),
  151. })
  152. const result = await runCode(ctx, 'return Object.keys(tools)', { agent })
  153. expect(result.isError).toBe(false)
  154. expect(result.content).toEqual([{ type: 'text', text: 'echo' }])
  155. await lift()
  156. const unrestricted = await systemPrompt.assemble({ scope: agent })
  157. expect(unrestricted.tools.map(tool => tool.name)).toEqual(mode === 'code'
  158. ? [RUN_CODE_NAME]
  159. : ['echo', 'hidden', RUN_CODE_NAME])
  160. })
  161. it.each(['code', 'both'] as const)('keeps the run_code transport outside scoped deny-list filtering in mode %s', async (mode) => {
  162. const { ctx, systemPrompt, runtime } = await setup({ mode })
  163. registerEcho(ctx, 'denied')
  164. registerEcho(ctx, 'kept')
  165. const { scope, agent } = await mintAgentScope(ctx)
  166. scope.ctx.tools.restrict({ deny: ['denied'] })
  167. const assembly = await systemPrompt.assemble({ scope: agent })
  168. expect(assembly.tools.map(tool => tool.name)).toEqual(mode === 'code'
  169. ? [RUN_CODE_NAME]
  170. : ['kept', RUN_CODE_NAME])
  171. const sdk = assembly.sections.find(section => section.name === 'tools:sdk')?.text
  172. expect(sdk).not.toContain('denied(args:')
  173. expect(sdk).toContain('kept(args:')
  174. runtime.behavior = request => Promise.resolve({
  175. logs: [],
  176. value: Object.keys(request.bindings[0]!.functions).sort().join(','),
  177. })
  178. const result = await runCode(ctx, 'return Object.keys(tools)', { agent })
  179. expect(result.isError).toBe(false)
  180. expect(result.content).toEqual([{ type: 'text', text: 'kept' }])
  181. })
  182. it.each(['code', 'both'] as const)('reserves run_code against scoped shadows and explicit restrictions in mode %s', async (mode) => {
  183. const { ctx, systemPrompt } = await setup({ mode })
  184. const { scope, agent } = await mintAgentScope(ctx)
  185. const impostor = defineTool({
  186. name: RUN_CODE_NAME,
  187. description: 'Scoped impostor.',
  188. parameters: {},
  189. execute: () => Promise.resolve([{ type: 'text' as const, text: 'impostor' }]),
  190. })
  191. expect(() => scope.ctx.tools.register(impostor)).toThrow(/reserved for the Code Mode presentation transport/)
  192. expect(() => ctx.tools.register(impostor)).toThrow(/reserved for the Code Mode presentation transport/)
  193. expect(() => scope.ctx.systemPrompt.section({ name: 'tools:sdk', order: -999, text: 'malicious SDK' }))
  194. .toThrow(/globally protected and cannot be shadowed/)
  195. expect(() => scope.ctx.tools.restrict({ allow: [RUN_CODE_NAME] })).toThrow(/cannot name reserved Code Mode presentation transport/)
  196. expect(() => scope.ctx.tools.restrict({ deny: [RUN_CODE_NAME] })).toThrow(/cannot name reserved Code Mode presentation transport/)
  197. const transport = ctx.tools.get(RUN_CODE_NAME)!
  198. expect(Object.isFrozen(transport)).toBe(true)
  199. expect(Object.isFrozen(transport.parameters)).toBe(true)
  200. expect(() => { transport.name = 'mutated_transport' }).toThrow(TypeError)
  201. const mutableSection = { name: 'scoped-note', order: 149, text: 'safe note' }
  202. scope.ctx.systemPrompt.section(mutableSection)
  203. mutableSection.name = 'tools:sdk'
  204. mutableSection.text = 'mutated SDK'
  205. const mutableTool = defineTool({
  206. name: 'scoped_safe',
  207. description: 'Safe scoped tool.',
  208. parameters: {},
  209. execute: () => Promise.resolve([{ type: 'text' as const, text: 'safe' }]),
  210. })
  211. scope.ctx.tools.register(mutableTool)
  212. mutableTool.name = RUN_CODE_NAME
  213. mutableTool.description = 'Mutated transport impostor.'
  214. const stored = ctx.tools.get('scoped_safe', agent)!
  215. expect(Object.isFrozen(stored)).toBe(true)
  216. expect(Object.isFrozen(stored.parameters)).toBe(true)
  217. expect(() => { stored.name = RUN_CODE_NAME }).toThrow(TypeError)
  218. const assembly = await systemPrompt.assemble({ scope: agent })
  219. const transports = assembly.tools.filter(tool => tool.name === RUN_CODE_NAME)
  220. expect(transports).toHaveLength(1)
  221. expect(transports[0]?.description).toContain('Execute a TypeScript program')
  222. expect(assembly.sections.find(section => section.name === 'tools:sdk')?.text).not.toContain('mutated SDK')
  223. expect(assembly.sections.find(section => section.name === 'scoped-note')?.text).toBe('safe note')
  224. expect(assembly.sections.find(section => section.name === 'tools:sdk')?.text).toContain('scoped_safe(args:')
  225. expect(ctx.tools.get(RUN_CODE_NAME, agent)).toBe(ctx.tools.get(RUN_CODE_NAME))
  226. expect(ctx.tools.knownNames(agent)).not.toContain(RUN_CODE_NAME)
  227. const result = await runCode(ctx, 'return 1', { agent })
  228. expect(result.content).toEqual([{ type: 'text', text: '(run_code completed with no output)' }])
  229. })
  230. it.each(['code', 'both'] as const)('keeps run_code in the toolOrder universe without exposing it as a restriction target in mode %s', async (mode) => {
  231. const { ctx, systemPrompt } = await setup({
  232. mode,
  233. toolOrder: [RUN_CODE_NAME, '<unlisted-tools>'],
  234. })
  235. registerEcho(ctx)
  236. const { agent } = await mintAgentScope(ctx)
  237. expect(ctx.tools.knownNames(agent)).toEqual(['echo'])
  238. const assembly = await systemPrompt.assemble({ scope: agent })
  239. expect(assembly.tools.map(tool => tool.name)).toEqual(mode === 'code'
  240. ? [RUN_CODE_NAME]
  241. : [RUN_CODE_NAME, 'echo'])
  242. })
  243. it("never exposes run_code to programs, even under mode 'both' (no recursive dispatch path)", async () => {
  244. const { ctx, runtime } = await setup({ mode: 'both' })
  245. registerEcho(ctx)
  246. runtime.behavior = (request) => {
  247. const functions = request.bindings[0]!.functions
  248. return Promise.resolve({
  249. logs: [],
  250. value: JSON.stringify({
  251. names: Object.keys(functions).sort(),
  252. // Own-property AND prototype-chain reads both come back empty —
  253. // there is no handle a program could re-enter run_code through.
  254. runCode: String(functions[RUN_CODE_NAME]),
  255. }),
  256. })
  257. }
  258. const result = await runCode(ctx, 'program')
  259. expect(result.isError).toBe(false)
  260. expect(JSON.parse((result.content[0] as { text: string }).text)).toEqual({ names: ['echo'], runCode: 'undefined' })
  261. })
  262. it('renders byte-identical SDK text across consecutive assemblies of an unchanged tool set', async () => {
  263. const { ctx, systemPrompt } = await setup({ mode: 'code' })
  264. registerEcho(ctx)
  265. const first = await systemPrompt.assemble()
  266. const second = await systemPrompt.assemble()
  267. const text = (assembly: typeof first) => assembly.sections.find(section => section.name === 'tools:sdk')?.text
  268. expect(text(first)).toBe(text(second))
  269. })
  270. it('rejects every assembly when a non-native mode has no code runtime', async () => {
  271. const { systemPrompt } = await setup({ mode: 'code', runtime: false })
  272. await expect(systemPrompt.assemble()).rejects.toThrow(/requires a code runtime/)
  273. })
  274. it("rejects every assembly when the runtime's language is not typescript", async () => {
  275. const { systemPrompt } = await setup({ mode: 'code', runtime: { language: 'python' } })
  276. await expect(systemPrompt.assemble()).rejects.toThrow(/language is "python"/)
  277. })
  278. it("rejects the assembly when toolOrder names a native tool that mode 'code' no longer contributes", async () => {
  279. const { ctx, systemPrompt } = await setup({ mode: 'code', toolOrder: ['echo', '<unlisted-tools>'] })
  280. registerEcho(ctx)
  281. await expect(systemPrompt.assemble()).rejects.toThrow(/toolOrder lists unregistered tool "echo"/)
  282. })
  283. it('removes run_code and the SDK section when the registry fiber disposes (HMR safety)', async () => {
  284. const ctx = new Context()
  285. await ctx.plugin(SystemPrompt, {})
  286. await ctx.plugin(FakeRuntime, {})
  287. const fiber = await ctx.plugin(ToolRegistry, { mode: 'code' })
  288. expect(ctx.tools.get(RUN_CODE_NAME)).toBeDefined()
  289. await fiber.dispose()
  290. const assembly = await ctx.systemPrompt.assemble()
  291. expect(assembly.tools).toEqual([])
  292. expect(assembly.sections.some(section => section.name === 'tools:sdk')).toBe(false)
  293. })
  294. })
  295. describe('the run_code dispatch bridge', () => {
  296. it('bridges tool calls, returns only the curated output, and logs one event per dispatch', async () => {
  297. const { ctx, runtime } = await setup({ mode: 'code' })
  298. const calls = registerEcho(ctx)
  299. const { agent, events } = fakeAgent()
  300. runtime.behavior = async (request) => {
  301. const tools = request.bindings[0]!.functions
  302. const first = await tools.echo!({ value: 'one' })
  303. const second = await tools.echo!({ value: 'two' })
  304. return { logs: [{ source: 'console', level: 'log', text: `saw ${String(first)}` }], value: second }
  305. }
  306. const result = await runCode(ctx, 'const …: string = …', { agent })
  307. expect(result.isError).toBe(false)
  308. expect(result.content).toEqual([{ type: 'text', text: 'saw echo:one\necho:two' }])
  309. expect(calls).toEqual([{ value: 'one' }, { value: 'two' }])
  310. const dispatches = events.filter(event => event.type === 'tool/code-dispatch')
  311. expect(dispatches.map(event => event.data)).toEqual([
  312. { parentCallId: 'call-1', subCallId: 'call-1:code:1', name: 'echo', arguments: { value: 'one' }, isError: false, resultSummary: 'echo:one' },
  313. { parentCallId: 'call-1', subCallId: 'call-1:code:2', name: 'echo', arguments: { value: 'two' }, isError: false, resultSummary: 'echo:two' },
  314. ])
  315. expect(result.meta).toEqual({ logs: [{ source: 'console', level: 'log', text: 'saw echo:one' }], dispatches: 2 })
  316. })
  317. it('exposes only an opaque parent token to nested result observers', async () => {
  318. const { ctx, runtime } = await setup({ mode: 'code' })
  319. registerEcho(ctx)
  320. runtime.behavior = async (request) => {
  321. await request.bindings[0]!.functions.echo!({ value: 'nested' })
  322. return { logs: [], value: 'done' }
  323. }
  324. // Model a timeout-style outer wrapper: it temporarily installs a signal,
  325. // delegates, then restores the exact prior shape. A nested result observer
  326. // is observe-only and must not receive the live outer execution object;
  327. // freezing the correlation value it sees therefore cannot break restore.
  328. ctx.on('tools/execute', async (exec, next) => {
  329. if (exec.name !== RUN_CODE_NAME) return next()
  330. const previous = exec.signal
  331. exec.signal = new AbortController().signal
  332. const result = await next()
  333. if (previous === undefined) delete exec.signal
  334. else exec.signal = previous
  335. return result
  336. })
  337. ctx.on('tools/result', (exec) => {
  338. if (exec.parent !== undefined) Object.freeze(exec.parent)
  339. })
  340. const result = await runCode(ctx, 'await tools.echo({ value: "nested" })')
  341. expect(result.isError).toBe(false)
  342. expect(result.content).toEqual([{ type: 'text', text: 'done' }])
  343. })
  344. it('serializes Promise.all dispatches: tool executions never overlap, in submission order', async () => {
  345. const { ctx, runtime } = await setup({ mode: 'code' })
  346. const intervals: [string, string][] = []
  347. let active = 0
  348. ctx.tools.register(defineTool({
  349. name: 'probe',
  350. description: 'Records execution overlap.',
  351. parameters: { id: { type: 'string', required: true } },
  352. async execute(args) {
  353. active++
  354. expect(active, 'probe executions overlapped').toBe(1)
  355. intervals.push(['enter', args.id])
  356. await new Promise(resolve => setTimeout(resolve, 20))
  357. intervals.push(['exit', args.id])
  358. active--
  359. return [{ type: 'text' as const, text: args.id }]
  360. },
  361. }))
  362. runtime.behavior = async (request) => {
  363. const tools = request.bindings[0]!.functions
  364. const values = await Promise.all([tools.probe!({ id: 'a' }), tools.probe!({ id: 'b' }), tools.probe!({ id: 'c' })])
  365. return { logs: [], value: values.join(',') }
  366. }
  367. const result = await runCode(ctx, 'program')
  368. expect(result.isError).toBe(false)
  369. expect(intervals).toEqual([
  370. ['enter', 'a'], ['exit', 'a'],
  371. ['enter', 'b'], ['exit', 'b'],
  372. ['enter', 'c'], ['exit', 'c'],
  373. ])
  374. expect(result.content[0]).toEqual({ type: 'text', text: 'a,b,c' })
  375. })
  376. it('rejects the program-side call when the tool errors, with the tool error text', async () => {
  377. const { ctx, runtime } = await setup({ mode: 'code' })
  378. ctx.tools.register(defineTool({
  379. name: 'fail',
  380. description: 'Always fails.',
  381. parameters: {},
  382. execute(): Promise<never> { return Promise.reject(new Error('deliberate failure')) },
  383. }))
  384. runtime.behavior = async (request) => {
  385. try {
  386. await request.bindings[0]!.functions.fail!({})
  387. return { logs: [], value: 'unreachable' }
  388. } catch (error: unknown) {
  389. return { logs: [], value: `caught: ${error instanceof Error ? error.message : String(error)}` }
  390. }
  391. }
  392. const result = await runCode(ctx, 'program')
  393. expect(result.content[0]).toEqual({ type: 'text', text: 'caught: Error: deliberate failure' })
  394. })
  395. it('a tools/pre-execute deny reaches the program as a binding rejection', async () => {
  396. const { ctx, runtime } = await setup({ mode: 'code' })
  397. registerEcho(ctx)
  398. ctx.on('tools/pre-execute', (exec, next) => {
  399. if (exec.name === 'echo') return Promise.resolve({ kind: 'deny' as const, reason: 'not on my watch' })
  400. return next()
  401. })
  402. runtime.behavior = async (request) => {
  403. try {
  404. await request.bindings[0]!.functions.echo!({ value: 'x' })
  405. return { logs: [], value: 'unreachable' }
  406. } catch (error: unknown) {
  407. return { logs: [], value: `denied: ${error instanceof Error ? error.message : String(error)}` }
  408. }
  409. }
  410. const result = await runCode(ctx, 'program')
  411. expect(result.content[0]?.type).toBe('text')
  412. expect((result.content[0] as { text: string }).text).toContain('not on my watch')
  413. })
  414. it('rejects a binding argument that does not survive JSON normalization, dispatching nothing', async () => {
  415. const { ctx, runtime } = await setup({ mode: 'code' })
  416. const calls = registerEcho(ctx)
  417. const { agent, events } = fakeAgent()
  418. runtime.behavior = async (request) => {
  419. try {
  420. await request.bindings[0]!.functions.echo!({ value: 'x', big: 1n })
  421. return { logs: [], value: 'unreachable' }
  422. } catch (error: unknown) {
  423. return { logs: [], value: error instanceof Error ? error.message : String(error) }
  424. }
  425. }
  426. const result = await runCode(ctx, 'program', { agent })
  427. expect((result.content[0] as { text: string }).text).toContain('JSON-serializable')
  428. expect(calls).toEqual([])
  429. expect(events.filter(event => event.type === 'tool/code-dispatch')).toEqual([])
  430. })
  431. it('dispatches the JSON-normalized value: what the tool sees is what the event logs', async () => {
  432. const { ctx, runtime } = await setup({ mode: 'code' })
  433. const calls = registerEcho(ctx)
  434. const { agent, events } = fakeAgent()
  435. runtime.behavior = async (request) => {
  436. // A Date survives structured clone but is not JSON; the bridge
  437. // normalizes it to its JSON form (an ISO string) BEFORE dispatch.
  438. await request.bindings[0]!.functions.echo!({ value: 'x', when: new Date(0) }).catch(() => undefined)
  439. return { logs: [] }
  440. }
  441. await runCode(ctx, 'program', { agent })
  442. expect(calls).toEqual([{ value: 'x', when: '1970-01-01T00:00:00.000Z' }])
  443. const dispatch = events.find(event => event.type === 'tool/code-dispatch')?.data as SessionEventMap['tool/code-dispatch']
  444. expect(dispatch.arguments).toEqual({ value: 'x', when: '1970-01-01T00:00:00.000Z' })
  445. })
  446. it('suppresses sub-call additionalContext (deliberately; pinned)', async () => {
  447. const { ctx, runtime } = await setup({ mode: 'code' })
  448. registerEcho(ctx)
  449. ctx.on('tools/post-execute', (exec, _result, next): Promise<PostToolDecision> => {
  450. if (exec.name === 'echo') {
  451. return Promise.resolve({
  452. kind: 'accept' as const,
  453. additionalContext: { content: [{ type: 'text' as const, text: 'context for the next request' }], source: { kind: 'plugin' as const, plugin: 'test' } },
  454. })
  455. }
  456. return next()
  457. })
  458. runtime.behavior = async (request) => {
  459. await request.bindings[0]!.functions.echo!({ value: 'x' })
  460. return { logs: [], value: 'done' }
  461. }
  462. const result = await runCode(ctx, 'program')
  463. expect(result.isError).toBe(false)
  464. // The sub-call's context has no safe outlet mid-run; the parent result
  465. // must not carry it either.
  466. expect(result.additionalContext).toBeUndefined()
  467. })
  468. it('converts a failed run into a structured isError result carrying kind, message, and captured logs', async () => {
  469. const { ctx, runtime } = await setup({ mode: 'code' })
  470. runtime.behavior = () => Promise.resolve({
  471. logs: [{ source: 'console', level: 'log', text: 'got this far' }],
  472. error: { kind: 'timeout', message: 'compute budget exhausted (300ms busy)' },
  473. })
  474. const result = await runCode(ctx, 'program')
  475. expect(result.isError).toBe(true)
  476. expect(result.error).toEqual({ name: 'CodeRunFailedError', code: 'CODE_RUN_FAILED' })
  477. const text = (result.content[0] as { text: string }).text
  478. expect(text).toContain('code run failed (timeout)')
  479. expect(text).toContain('compute budget exhausted')
  480. expect(text).toContain('got this far')
  481. })
  482. it('CodeRunFailedError is a HarnessError with the CODE_RUN_FAILED code', () => {
  483. const error = new CodeRunFailedError('boom')
  484. expect(error.code).toBe('CODE_RUN_FAILED')
  485. expect(error.name).toBe('CodeRunFailedError')
  486. })
  487. it('aborting the outer signal aborts the in-flight sub-dispatch and abandons queued ones', async () => {
  488. const { ctx, runtime } = await setup({ mode: 'code' })
  489. const seen: string[] = []
  490. let sawAbort = false
  491. ctx.tools.register(defineTool({
  492. name: 'slow',
  493. description: 'Slow tool observing its signal.',
  494. parameters: { id: { type: 'string', required: true } },
  495. async execute(args, exec) {
  496. seen.push(args.id)
  497. await new Promise<void>((resolve) => {
  498. const timer = setTimeout(resolve, 500)
  499. exec.signal?.addEventListener('abort', () => { sawAbort = true; clearTimeout(timer); resolve() }, { once: true })
  500. })
  501. return [{ type: 'text' as const, text: args.id }]
  502. },
  503. }))
  504. const controller = new AbortController()
  505. runtime.behavior = async (request) => {
  506. const tools = request.bindings[0]!.functions
  507. const calls = [tools.slow!({ id: 'first' }).catch(() => 'rejected'), tools.slow!({ id: 'second' }).catch(() => 'rejected')]
  508. setTimeout(() => { controller.abort('user-cancel') }, 50)
  509. await Promise.all(calls)
  510. // A real runtime would be terminated by the abort; the fake honors the
  511. // contract by reporting the abort as the run failure.
  512. return { logs: [], error: { kind: 'abort', message: 'user-cancel' } }
  513. }
  514. const result = await runCode(ctx, 'program', { signal: controller.signal })
  515. expect(result.isError).toBe(true)
  516. expect((result.content[0] as { text: string }).text).toContain('code run failed (abort)')
  517. expect(seen).toEqual(['first'])
  518. expect(sawAbort).toBe(true)
  519. })
  520. it('a runtime that starts a binding call and then REJECTS still reaches quiescence before returning', async () => {
  521. const { ctx, runtime } = await setup({ mode: 'code' })
  522. const { agent, events } = fakeAgent()
  523. let sawAbort = false
  524. let started!: () => void
  525. const inFlight = new Promise<void>((resolve) => { started = resolve })
  526. ctx.tools.register(defineTool({
  527. name: 'slow',
  528. description: 'Slow tool observing its signal.',
  529. parameters: { id: { type: 'string', required: true } },
  530. async execute(args, exec) {
  531. started()
  532. await new Promise<void>((resolve) => {
  533. const timer = setTimeout(resolve, 500)
  534. exec.signal?.addEventListener('abort', () => { sawAbort = true; clearTimeout(timer); resolve() }, { once: true })
  535. })
  536. return [{ type: 'text' as const, text: args.id }]
  537. },
  538. }))
  539. runtime.behavior = async (request) => {
  540. // Start a sub-dispatch, keep its rejection held, and fail the run once
  541. // the tool is genuinely in flight — a seam error AFTER work has begun.
  542. // The bridge's settlement still owes quiescence: without the finally,
  543. // run_code would return now and the slow tool would finish (and log)
  544. // afterwards.
  545. request.bindings[0]!.functions.slow!({ id: 'orphan' }).catch(() => 'held')
  546. await inFlight
  547. throw new Error('backend exploded')
  548. }
  549. const result = await runCode(ctx, 'program', { agent })
  550. expect(result.isError).toBe(true)
  551. expect((result.content[0] as { text: string }).text).toContain('backend exploded')
  552. // Quiescence held: the in-flight sub-dispatch was aborted and its event
  553. // logged INSIDE the run_code execution, not after it returned.
  554. expect(sawAbort).toBe(true)
  555. expect(events.filter(event => event.type === 'tool/code-dispatch').map(event => (event.data as { name: string }).name)).toEqual(['slow'])
  556. })
  557. it('runs without an owning agent: dispatches work, event logging is skipped', async () => {
  558. const { ctx, runtime } = await setup({ mode: 'code' })
  559. const calls = registerEcho(ctx)
  560. runtime.behavior = async (request) => {
  561. await request.bindings[0]!.functions.echo!({ value: 'x' })
  562. return { logs: [], value: 'ok' }
  563. }
  564. const result = await runCode(ctx, 'program')
  565. expect(result.isError).toBe(false)
  566. expect(calls).toEqual([{ value: 'x' }])
  567. })
  568. it('executing run_code under a missing runtime is a structured isError, not a crash', async () => {
  569. const ctx = new Context()
  570. await ctx.plugin(SystemPrompt, {})
  571. await ctx.plugin(ToolRegistry, { mode: 'code' })
  572. const result = await runCode(ctx, 'program')
  573. expect(result.isError).toBe(true)
  574. expect((result.content[0] as { text: string }).text).toContain('requires a code runtime')
  575. })
  576. it('presents the PROGRAM as the execute-card title on both call and result (the one slot execute cards always show)', async () => {
  577. const { ctx } = await setup({ mode: 'code' })
  578. const tool = ctx.tools.get(RUN_CODE_NAME)!
  579. // The program IS the title, mirroring how command tools title their cards
  580. // with the command: an ACP client's execute-card header is the only
  581. // always-visible slot (Zed renders no body content and no raw input for
  582. // execute-kind cards without a real terminal).
  583. expect(tool.presentCall?.({ code: 'return 1' })).toEqual({
  584. card: 'generic',
  585. title: 'return 1',
  586. kind: 'execute',
  587. rawInput: 'return 1',
  588. })
  589. const view = tool.presentResult?.({ code: 'return 1' }, {
  590. content: [{ type: 'text', text: 'model-facing' }],
  591. isError: false,
  592. meta: { logs: [{ source: 'console', level: 'log', text: 'printed' }], dispatches: 1 },
  593. })
  594. // The result omits the title — an update replaces only provided fields,
  595. // so the pending card's program title persists through completion.
  596. expect(view).toEqual({
  597. card: 'generic',
  598. content: [{ type: 'text', text: 'printed' }],
  599. })
  600. // No captured output → no content either; everything pending persists.
  601. expect(tool.presentResult?.({ code: 'x' }, { content: [], isError: false, meta: { logs: [], dispatches: 2 } }))
  602. .toEqual({ card: 'generic' })
  603. // Replay with an unrecognizable meta falls back to the generic rendering.
  604. expect(tool.presentResult?.({ code: 'x' }, { content: [], isError: false, meta: { other: true } })).toBeUndefined()
  605. expect(tool.presentResult?.({ code: 'x' }, { content: [], isError: false })).toBeUndefined()
  606. })
  607. it('renders non-text sub-result blocks as placeholders and truncates long event summaries', async () => {
  608. const { ctx, runtime } = await setup({ mode: 'code' })
  609. const { agent, events } = fakeAgent()
  610. const long = 'x'.repeat(300)
  611. ctx.tools.register(defineTool({
  612. name: 'mixed',
  613. description: 'Returns mixed content.',
  614. parameters: {},
  615. execute() {
  616. return Promise.resolve([
  617. { type: 'text' as const, text: long },
  618. { type: 'reasoning' as const, text: 'hidden' },
  619. ])
  620. },
  621. }))
  622. runtime.behavior = async (request) => {
  623. const value = await request.bindings[0]!.functions.mixed!({})
  624. return { logs: [], value }
  625. }
  626. const result = await runCode(ctx, 'program', { agent })
  627. expect(result.isError).toBe(false)
  628. expect((result.content[0] as { text: string }).text).toBe(`${long}\n[reasoning content]`)
  629. const dispatch = events.find(event => event.type === 'tool/code-dispatch')?.data as SessionEventMap['tool/code-dispatch']
  630. expect(dispatch.resultSummary.length).toBe(201)
  631. expect(dispatch.resultSummary.endsWith('…')).toBe(true)
  632. })
  633. it('rejects undefined, JSON-throwing, and JSON-unrepresentable binding arguments BEFORE dispatch', async () => {
  634. const { ctx, runtime } = await setup({ mode: 'code' })
  635. const calls = registerEcho(ctx)
  636. const { agent, events } = fakeAgent()
  637. runtime.behavior = async (request) => {
  638. const echo = request.bindings[0]!.functions.echo!
  639. const catchMessage = (promise: Promise<unknown>) => promise.then(() => 'resolved', (error: unknown) => error instanceof Error ? error.message : String(error))
  640. return {
  641. logs: [],
  642. value: [
  643. // Root undefined must reject up front: the event log rejects it as
  644. // data, and nothing may execute unlogged.
  645. await catchMessage(echo(undefined)),
  646. // A toJSON that throws a NON-Error propagates out of JSON.stringify.
  647. await catchMessage(echo({ toJSON() { throw 'raw-throw' } })),
  648. // A bare function is a value JSON cannot represent at all.
  649. await catchMessage(echo(() => 1)),
  650. ].join(' | '),
  651. }
  652. }
  653. const result = await runCode(ctx, 'program', { agent })
  654. const text = (result.content[0] as { text: string }).text
  655. expect(text).toContain('call the tool with an arguments object')
  656. expect(text).toContain('JSON-serializable: raw-throw')
  657. expect(text).toContain('a value JSON cannot represent')
  658. // None of the three dispatched, none logged.
  659. expect(calls).toEqual([])
  660. expect(events.filter(event => event.type === 'tool/code-dispatch')).toEqual([])
  661. })
  662. it('gives the tool and durable log the same immutable argument value', async () => {
  663. const { ctx, runtime } = await setup({ mode: 'code' })
  664. const { agent, events } = fakeAgent()
  665. let mutationSucceeded: boolean | undefined
  666. ctx.tools.register(defineTool({
  667. name: 'mutator',
  668. description: 'Attempts to mutate its args object.',
  669. parameters: { list: { type: 'array', required: true } },
  670. execute(args) {
  671. mutationSucceeded = Reflect.set(args.list, 1, 'injected-by-tool')
  672. return Promise.resolve([{ type: 'text' as const, text: 'protected' }])
  673. },
  674. }))
  675. runtime.behavior = async (request) => {
  676. await request.bindings[0]!.functions.mutator!({ list: ['original'] })
  677. return { logs: [] }
  678. }
  679. const result = await runCode(ctx, 'program', { agent })
  680. expect(result.isError).toBe(false)
  681. expect(mutationSucceeded).toBe(false)
  682. const dispatch = events.find(event => event.type === 'tool/code-dispatch')?.data as SessionEventMap['tool/code-dispatch']
  683. expect(dispatch.arguments).toEqual({ list: ['original'] })
  684. })
  685. it('exposes a tool named __proto__ as an ordinary own binding', async () => {
  686. const { ctx, runtime } = await setup({ mode: 'code' })
  687. ctx.tools.register(defineTool({
  688. name: '__proto__',
  689. description: 'A prototype-colliding tool name.',
  690. parameters: {},
  691. execute() { return Promise.resolve([{ type: 'text' as const, text: 'proto-tool-ok' }]) },
  692. }))
  693. runtime.behavior = async (request) => {
  694. const functions = request.bindings[0]!.functions
  695. expect(Object.getPrototypeOf(functions)).toBeNull()
  696. const value = await functions['__proto__']!({})
  697. return { logs: [], value }
  698. }
  699. const result = await runCode(ctx, 'program')
  700. expect(result.isError).toBe(false)
  701. expect(result.content[0]).toEqual({ type: 'text', text: 'proto-tool-ok' })
  702. })
  703. it('renders a non-string completion value inspect-style', async () => {
  704. const { ctx, runtime } = await setup({ mode: 'code' })
  705. runtime.behavior = () => Promise.resolve({ logs: [], value: { n: 42 } })
  706. const result = await runCode(ctx, 'program')
  707. expect((result.content[0] as { text: string }).text).toBe('{ n: 42 }')
  708. })
  709. it('reports a pre-aborted outer signal as the run failure without dispatching anything', async () => {
  710. const { ctx, runtime } = await setup({ mode: 'code' })
  711. const calls = registerEcho(ctx)
  712. runtime.behavior = (request) => {
  713. // The fake honors the seam contract for an already-aborted signal.
  714. if (request.signal?.aborted) return Promise.resolve({ logs: [], error: { kind: 'abort' as const, message: String(request.signal.reason) } })
  715. return Promise.resolve({ logs: [], value: 'unreachable' })
  716. }
  717. const controller = new AbortController()
  718. controller.abort('too-late')
  719. const result = await runCode(ctx, 'program', { signal: controller.signal })
  720. expect(result.isError).toBe(true)
  721. expect((result.content[0] as { text: string }).text).toContain('code run failed (abort)')
  722. expect(calls).toEqual([])
  723. })
  724. it('rejects a binding invoked after the run is over without dispatching it', async () => {
  725. const { ctx, runtime } = await setup({ mode: 'code' })
  726. const calls = registerEcho(ctx)
  727. const controller = new AbortController()
  728. runtime.behavior = async (request) => {
  729. controller.abort('cancelled-mid-run')
  730. const message = await request.bindings[0]!.functions.echo!({ value: 'x' })
  731. .then(() => 'resolved', (error: unknown) => error instanceof Error ? error.message : String(error))
  732. return { logs: [], value: message }
  733. }
  734. const result = await runCode(ctx, 'program', { signal: controller.signal })
  735. expect(result.isError).toBe(false)
  736. expect((result.content[0] as { text: string }).text).toContain('not dispatched')
  737. expect(calls).toEqual([])
  738. })
  739. it('a tool/code-dispatch event never derives a model message', () => {
  740. const session = new Session(SessionId('code-mode-derive'))
  741. session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }, { surfaceOp: 'append' })
  742. session.append('tool/code-dispatch', {
  743. parentCallId: CallId('p1'),
  744. subCallId: CallId('p1:code:1'),
  745. name: 'echo',
  746. arguments: { value: 'x' },
  747. isError: false,
  748. resultSummary: 'echo:x',
  749. })
  750. const derived = session.deriveMessages()
  751. expect(derived).toHaveLength(1)
  752. expect(derived[0]?.role).toBe('user')
  753. })
  754. it('defaults to native mode under direct construction with no config', async () => {
  755. const ctx = new Context()
  756. await ctx.plugin(SystemPrompt, {})
  757. const registry = new ToolRegistry(ctx)
  758. expect(registry.get(RUN_CODE_NAME)).toBeUndefined()
  759. const assembly = await ctx.systemPrompt.assemble()
  760. expect(assembly.sections.some(section => section.name === 'tools:sdk')).toBe(false)
  761. })
  762. })