code-mode.spec.ts 70 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542
  1. import { describe, expect, it } from 'vitest'
  2. import { Context } from 'cordis'
  3. import { createUserMessage, CallId } from '@deepseek-ai/dsh-llm'
  4. import { createScope } from '@deepseek-ai/dsh-scope'
  5. import type { Scope } from '@deepseek-ai/dsh-scope'
  6. import SystemPrompt from '@deepseek-ai/dsh-system-prompt'
  7. import { CodeRuntime } from '@deepseek-ai/dsh-code-runtime'
  8. import type { CodeRunRequest, CodeRunResult } from '@deepseek-ai/dsh-code-runtime'
  9. import ToolRegistry, { CodeRunFailedError, RUN_CODE_NAME, TOOL_ABORTED_BEFORE_DISPATCH, defineContentToolFixture, defineTool } from '@deepseek-ai/dsh-tools'
  10. import type { Config, JsonSchemaNode, PostToolDecision, ToolExecutionResult } from '@deepseek-ai/dsh-tools'
  11. import type { Agent } from '@deepseek-ai/dsh-agent'
  12. import { Session, SessionId } from '@deepseek-ai/dsh-session'
  13. import type { JsonValue, SessionEventMap } from '@deepseek-ai/dsh-session'
  14. const testToolSignal = new AbortController().signal
  15. /**
  16. * Code Mode unit tier (per the Agent Note's plan): provider contribution per mode,
  17. * misconfiguration rejections, the run_code dispatch bridge (serialization,
  18. * abort, JSON normalization, error mapping, events, quiescence), and HMR
  19. * safety — all against an in-repo fake runtime, exactly the
  20. * interface/implementation/consumer shape the seam promises.
  21. */
  22. /** A scriptable in-repo CodeRuntime: each test sets `behavior` to drive the bindings however it needs. */
  23. class FakeRuntime extends CodeRuntime {
  24. readonly language: string
  25. readonly isolation = 'fake'
  26. behavior: (request: CodeRunRequest) => Promise<CodeRunResult> = () => Promise.resolve({ logs: [] })
  27. lastRequest?: CodeRunRequest
  28. constructor(ctx: Context, config: { language?: string } = {}) {
  29. super(ctx)
  30. this.language = config.language ?? 'typescript'
  31. }
  32. run(request: CodeRunRequest): Promise<CodeRunResult> {
  33. this.lastRequest = request
  34. return this.behavior(request)
  35. }
  36. }
  37. interface SetupOptions {
  38. mode?: Config['mode']
  39. maxParallelSubCalls?: number
  40. runtime?: false | { language?: string }
  41. toolOrder?: string[]
  42. }
  43. async function setup(options: SetupOptions = {}) {
  44. const ctx = new Context()
  45. await ctx.plugin(SystemPrompt, { ...options.toolOrder ? { toolOrder: options.toolOrder } : {} })
  46. await ctx.plugin(ToolRegistry, { mode: options.mode ?? 'code', ...options.maxParallelSubCalls !== undefined ? { maxParallelSubCalls: options.maxParallelSubCalls } : {} })
  47. let runtime: FakeRuntime | undefined
  48. if (options.runtime !== false) {
  49. await ctx.plugin(FakeRuntime, options.runtime ?? {})
  50. runtime = ctx.codeRuntime as FakeRuntime
  51. }
  52. return { ctx, tools: ctx.tools, systemPrompt: ctx.systemPrompt, runtime: runtime! }
  53. }
  54. /** Mint one production-shaped agent scope that can register scoped tool policy. */
  55. async function mintAgentScope(ctx: Context, name = 'scoped'): Promise<{ scope: Scope; agent: Agent }> {
  56. const agent = { id: SessionId(name) } as Agent
  57. let scope!: Scope
  58. await ctx.plugin(Object.assign((inner: Context) => { scope = createScope(inner, agent) },
  59. { inject: ['tools', 'systemPrompt'] }))
  60. return { scope, agent }
  61. }
  62. /** Register a trivial echo tool; returns the calls it received. */
  63. function registerEcho(ctx: Context, name = 'echo'): unknown[] {
  64. const calls: unknown[] = []
  65. ctx.tools.register(defineTool({
  66. name,
  67. description: `Echo tool ${name}.`,
  68. parameters: { value: { type: 'string', required: true } },
  69. output: {
  70. schema: { type: 'string' },
  71. render: (_args, value) => [{ type: 'text', text: value }],
  72. },
  73. execute(args) {
  74. calls.push(args)
  75. return Promise.resolve(`${name}:${args.value}`)
  76. },
  77. }))
  78. return calls
  79. }
  80. /** A structural fake of the owning agent: captures session appends. */
  81. function fakeAgent(): { agent: Agent; events: { type: string; data: unknown }[] } {
  82. const events: { type: string; data: unknown }[] = []
  83. const agent = {
  84. session: {
  85. header: { cwd: '/workspace' },
  86. append: (type: string, data: unknown) => { events.push({ type, data }) },
  87. },
  88. } as unknown as Agent
  89. return { agent, events }
  90. }
  91. /** Dispatch run_code through the registry pipeline, as the loop would. */
  92. async function runCode(
  93. ctx: Context,
  94. code: string,
  95. extras: { agent?: Agent; signal?: AbortSignal; description?: string } = {},
  96. ): Promise<ToolExecutionResult> {
  97. return ctx.tools.execute({
  98. signal: testToolSignal,
  99. callId: CallId('call-1'),
  100. name: RUN_CODE_NAME,
  101. arguments: { code, description: extras.description ?? 'Run the test program' },
  102. ...extras.agent ? { agent: extras.agent } : {},
  103. ...extras.signal ? { signal: extras.signal } : {},
  104. })
  105. }
  106. describe('mode-aware wire contribution', () => {
  107. it("mode 'native' contributes every schema, no run_code, no SDK section — and needs no runtime", async () => {
  108. const { ctx, systemPrompt } = await setup({ mode: 'native', runtime: false })
  109. registerEcho(ctx)
  110. const assembly = await systemPrompt.assemble()
  111. expect(assembly.tools.map(tool => tool.name)).toEqual(['echo'])
  112. expect(assembly.sections.some(section => section.name === 'tools:sdk')).toBe(false)
  113. })
  114. it("mode 'code' contributes exactly [run_code] plus the SDK section declaring the other tools", async () => {
  115. const { ctx, systemPrompt } = await setup({ mode: 'code' })
  116. registerEcho(ctx)
  117. const assembly = await systemPrompt.assemble()
  118. expect(assembly.tools.map(tool => tool.name)).toEqual([RUN_CODE_NAME])
  119. const sdk = assembly.sections.find(section => section.name === 'tools:sdk')
  120. expect(sdk?.text).toContain('declare const tools: {')
  121. expect(sdk?.text).toContain('echo: {')
  122. expect(sdk?.text).not.toContain('run_code:')
  123. })
  124. it('projects deeply nested output schemas into the Code Mode SDK without structured-clone recursion', async () => {
  125. const { ctx, systemPrompt } = await setup({ mode: 'code' })
  126. let output: JsonSchemaNode = { type: 'string' }
  127. for (let depth = 0; depth < 5_000; depth++) {
  128. output = { oneOf: [output, { type: 'null' }] }
  129. }
  130. ctx.tools.register({
  131. name: 'deep_output',
  132. description: 'Return a deeply nested output union.',
  133. parameters: { type: 'object', properties: {} },
  134. output: {
  135. schema: output,
  136. render: (_args, value) => [{ type: 'text', text: typeof value === 'string' ? value : 'null' }],
  137. },
  138. execute() { return Promise.resolve('ok') },
  139. })
  140. const assembly = await systemPrompt.assemble()
  141. const sdk = assembly.sections.find(section => section.name === 'tools:sdk')?.text
  142. expect(sdk).toContain('deep_output: Record<string, JsonValue>;')
  143. expect(sdk).toContain('deep_output: string | null')
  144. })
  145. it.each(['code', 'both'] as const)('treats expert assembly output as authoritative in mode %s', async (mode) => {
  146. const { ctx, systemPrompt } = await setup({ mode })
  147. registerEcho(ctx)
  148. ctx.on('system-prompt/assemble', async (_assembly, _context, next) => {
  149. const assembly = await next()
  150. return {
  151. ...assembly,
  152. sections: assembly.sections.filter(section => section.name !== 'tools:sdk'),
  153. tools: assembly.tools.filter(tool => tool.name !== RUN_CODE_NAME),
  154. }
  155. }, { prepend: true })
  156. const assembly = await systemPrompt.assemble()
  157. expect(assembly.sections.some(section => section.name === 'tools:sdk')).toBe(false)
  158. expect(assembly.tools.some(tool => tool.name === RUN_CODE_NAME)).toBe(false)
  159. })
  160. it.each(['code', 'both'] as const)('lets one scope shadow the default SDK section in mode %s', async (mode) => {
  161. const { ctx, systemPrompt } = await setup({ mode })
  162. registerEcho(ctx)
  163. const { scope, agent } = await mintAgentScope(ctx)
  164. scope.ctx.systemPrompt.section({ name: 'tools:sdk', order: 150, text: 'SCOPED SDK' })
  165. const scoped = await systemPrompt.assemble({ scope: agent })
  166. const global = await systemPrompt.assemble()
  167. expect(scoped.sections.find(section => section.name === 'tools:sdk')?.text).toBe('SCOPED SDK')
  168. expect(global.sections.find(section => section.name === 'tools:sdk')?.text).toContain('declare const tools:')
  169. })
  170. it("mode 'both' contributes every native schema plus run_code, and the SDK section", async () => {
  171. const { ctx, systemPrompt } = await setup({ mode: 'both' })
  172. registerEcho(ctx)
  173. const assembly = await systemPrompt.assemble()
  174. expect(assembly.tools.map(tool => tool.name)).toEqual(['echo', RUN_CODE_NAME])
  175. expect(assembly.sections.some(section => section.name === 'tools:sdk')).toBe(true)
  176. })
  177. it.each(['code', 'both'] as const)('keeps the run_code transport outside scoped allow-list filtering in mode %s', async (mode) => {
  178. const { ctx, systemPrompt, runtime } = await setup({ mode })
  179. registerEcho(ctx, 'echo')
  180. registerEcho(ctx, 'hidden')
  181. const { scope, agent } = await mintAgentScope(ctx)
  182. const lift = scope.ctx.tools.restrict({ allow: ['echo'] })
  183. const assembly = await systemPrompt.assemble({ scope: agent })
  184. expect(assembly.tools.map(tool => tool.name)).toEqual(mode === 'code'
  185. ? [RUN_CODE_NAME]
  186. : ['echo', RUN_CODE_NAME])
  187. const sdk = assembly.sections.find(section => section.name === 'tools:sdk')?.text
  188. expect(sdk).toContain('echo: {')
  189. expect(sdk).not.toContain('hidden:')
  190. runtime.behavior = request => Promise.resolve({
  191. logs: [],
  192. value: Object.keys(request.bindings[0]!.functions).sort().join(','),
  193. })
  194. const result = await runCode(ctx, 'return Object.keys(tools)', { agent })
  195. expect(result.isError).toBe(false)
  196. expect(result.content).toEqual([{ type: 'text', text: 'echo' }])
  197. lift()
  198. const unrestricted = await systemPrompt.assemble({ scope: agent })
  199. expect(unrestricted.tools.map(tool => tool.name)).toEqual(mode === 'code'
  200. ? [RUN_CODE_NAME]
  201. : ['echo', 'hidden', RUN_CODE_NAME])
  202. })
  203. it.each(['code', 'both'] as const)('keeps the run_code transport outside scoped deny-list filtering in mode %s', async (mode) => {
  204. const { ctx, systemPrompt, runtime } = await setup({ mode })
  205. registerEcho(ctx, 'denied')
  206. registerEcho(ctx, 'kept')
  207. const { scope, agent } = await mintAgentScope(ctx)
  208. scope.ctx.tools.restrict({ deny: ['denied'] })
  209. const assembly = await systemPrompt.assemble({ scope: agent })
  210. expect(assembly.tools.map(tool => tool.name)).toEqual(mode === 'code'
  211. ? [RUN_CODE_NAME]
  212. : ['kept', RUN_CODE_NAME])
  213. const sdk = assembly.sections.find(section => section.name === 'tools:sdk')?.text
  214. expect(sdk).not.toContain('denied:')
  215. expect(sdk).toContain('kept: {')
  216. runtime.behavior = request => Promise.resolve({
  217. logs: [],
  218. value: Object.keys(request.bindings[0]!.functions).sort().join(','),
  219. })
  220. const result = await runCode(ctx, 'return Object.keys(tools)', { agent })
  221. expect(result.isError).toBe(false)
  222. expect(result.content).toEqual([{ type: 'text', text: 'kept' }])
  223. })
  224. it.each(['code', 'both'] as const)('reserves run_code against scoped shadows and explicit restrictions in mode %s', async (mode) => {
  225. const { ctx, systemPrompt } = await setup({ mode })
  226. const { scope, agent } = await mintAgentScope(ctx)
  227. const impostor = defineContentToolFixture({
  228. name: RUN_CODE_NAME,
  229. description: 'Scoped impostor.',
  230. parameters: {},
  231. execute: () => Promise.resolve([{ type: 'text' as const, text: 'impostor' }]),
  232. })
  233. expect(() => scope.ctx.tools.register(impostor)).toThrow(/reserved for the Code Mode presentation transport/)
  234. expect(() => ctx.tools.register(impostor)).toThrow(/reserved for the Code Mode presentation transport/)
  235. expect(() => scope.ctx.tools.restrict({ allow: [RUN_CODE_NAME] })).toThrow(/cannot name reserved Code Mode presentation transport/)
  236. expect(() => scope.ctx.tools.restrict({ deny: [RUN_CODE_NAME] })).toThrow(/cannot name reserved Code Mode presentation transport/)
  237. scope.ctx.systemPrompt.section({ name: 'scoped-note', order: 149, text: 'safe note' })
  238. scope.ctx.tools.register(defineContentToolFixture({
  239. name: 'scoped_safe',
  240. description: 'Safe scoped tool.',
  241. parameters: {},
  242. execute: () => Promise.resolve([{ type: 'text' as const, text: 'safe' }]),
  243. }))
  244. const assembly = await systemPrompt.assemble({ scope: agent })
  245. const transports = assembly.tools.filter(tool => tool.name === RUN_CODE_NAME)
  246. expect(transports).toHaveLength(1)
  247. expect(transports[0]?.description).toContain('Execute a TypeScript program')
  248. expect(assembly.sections.find(section => section.name === 'scoped-note')?.text).toBe('safe note')
  249. expect(assembly.sections.find(section => section.name === 'tools:sdk')?.text).toContain('scoped_safe:')
  250. expect(ctx.tools.get(RUN_CODE_NAME, agent)).toBe(ctx.tools.get(RUN_CODE_NAME))
  251. const result = await runCode(ctx, 'return 1', { agent })
  252. expect(result.content).toEqual([{ type: 'text', text: '(run_code completed with no output)' }])
  253. })
  254. it.each(['code', 'both'] as const)('keeps run_code in the toolOrder universe without exposing it as a restriction target in mode %s', async (mode) => {
  255. const { ctx, systemPrompt } = await setup({
  256. mode,
  257. toolOrder: [RUN_CODE_NAME, '<unlisted-tools>'],
  258. })
  259. registerEcho(ctx)
  260. const { agent } = await mintAgentScope(ctx)
  261. const assembly = await systemPrompt.assemble({ scope: agent })
  262. expect(assembly.tools.map(tool => tool.name)).toEqual(mode === 'code'
  263. ? [RUN_CODE_NAME]
  264. : [RUN_CODE_NAME, 'echo'])
  265. })
  266. it("never exposes run_code to programs, even under mode 'both' (no recursive dispatch path)", async () => {
  267. const { ctx, runtime } = await setup({ mode: 'both' })
  268. registerEcho(ctx)
  269. runtime.behavior = (request) => {
  270. expect(request.bindings[0]!.errorClass).toEqual({
  271. name: 'ToolCallError',
  272. memberNameProperty: 'toolName',
  273. })
  274. const functions = request.bindings[0]!.functions
  275. return Promise.resolve({
  276. logs: [],
  277. value: JSON.stringify({
  278. names: Object.keys(functions).sort(),
  279. // Own-property AND prototype-chain reads both come back empty —
  280. // there is no handle a program could re-enter run_code through.
  281. runCode: String(functions[RUN_CODE_NAME]),
  282. }),
  283. })
  284. }
  285. const result = await runCode(ctx, 'program')
  286. expect(result.isError).toBe(false)
  287. expect(JSON.parse((result.content[0] as { text: string }).text)).toEqual({ names: ['echo'], runCode: 'undefined' })
  288. })
  289. it('renders byte-identical SDK text across consecutive assemblies of an unchanged tool set', async () => {
  290. const { ctx, systemPrompt } = await setup({ mode: 'code' })
  291. registerEcho(ctx)
  292. const first = await systemPrompt.assemble()
  293. const second = await systemPrompt.assemble()
  294. const text = (assembly: typeof first) => assembly.sections.find(section => section.name === 'tools:sdk')?.text
  295. expect(text(first)).toBe(text(second))
  296. })
  297. it('rejects every assembly when a non-native mode has no code runtime', async () => {
  298. const { systemPrompt } = await setup({ mode: 'code', runtime: false })
  299. await expect(systemPrompt.assemble()).rejects.toThrow(/requires a code runtime/)
  300. })
  301. it('rejects every assembly when the runtime language has no registered SDK renderer', async () => {
  302. const { systemPrompt } = await setup({ mode: 'code', runtime: { language: 'ruby' } })
  303. await expect(systemPrompt.assemble()).rejects.toThrow(/no SDK renderer registered for runtime language "ruby"/)
  304. })
  305. it('assembles under a python runtime by picking the Python SDK renderer', async () => {
  306. const { ctx, systemPrompt } = await setup({ mode: 'code', runtime: { language: 'python' } })
  307. registerEcho(ctx)
  308. const assembly = await systemPrompt.assemble()
  309. const sdk = assembly.sections.find(section => section.name === 'tools:sdk')
  310. expect(sdk?.text).toContain('class Tools(Protocol):')
  311. expect(sdk?.text).toContain('async def echo(self, args:')
  312. expect(sdk?.text).toContain('top-level `await`')
  313. })
  314. it('emits a TypeScript-flavored run_code schema under a typescript runtime', async () => {
  315. const { ctx, systemPrompt } = await setup({ mode: 'code', runtime: { language: 'typescript' } })
  316. registerEcho(ctx)
  317. const assembly = await systemPrompt.assemble()
  318. const runCodeSchema = assembly.tools.find(tool => tool.name === RUN_CODE_NAME)
  319. expect(runCodeSchema?.description).toContain('Execute a TypeScript program')
  320. expect(runCodeSchema?.description).toContain('BODY of an')
  321. const codeParam = (runCodeSchema?.parameters as { properties: { code: { description: string } } }).properties.code
  322. expect(codeParam.description).toBe('The program: the body of an async TypeScript function.')
  323. })
  324. it('emits a Python-flavored run_code schema under a python runtime (matches the SDK language)', async () => {
  325. const { ctx, systemPrompt } = await setup({ mode: 'code', runtime: { language: 'python' } })
  326. registerEcho(ctx)
  327. const assembly = await systemPrompt.assemble()
  328. const runCodeSchema = assembly.tools.find(tool => tool.name === RUN_CODE_NAME)
  329. expect(runCodeSchema?.description).toContain('Execute a Python program')
  330. expect(runCodeSchema?.description).toContain('`return <value>`')
  331. expect(runCodeSchema?.description).not.toContain('TypeScript')
  332. const codeParam = (runCodeSchema?.parameters as { properties: { code: { description: string } } }).properties.code
  333. expect(codeParam.description).toBe('The program: the body of an async Python function.')
  334. })
  335. it('resolves the run_code schema flavor lazily and fails loud on a language absent from the flavor table', async () => {
  336. // The flavor getter reads the runtime directly (peekRuntime), so it — not
  337. // requireCodeRuntime — owns the flavor-table guard. A language with no
  338. // flavor entry throws when the schema is projected, keeping
  339. // RUN_CODE_FLAVORS coupled to SDK_RENDERERS. Assembly's requireCodeRuntime
  340. // rejects such a language earlier; this reaches the guard on its own.
  341. const { ctx } = await setup({ mode: 'code', runtime: { language: 'ruby' } })
  342. const definition = ctx.tools.get(RUN_CODE_NAME)
  343. // Names the known languages, symmetric with the SDK_RENDERERS guard: this
  344. // is the reachable rejection, so it must be at least as diagnosable.
  345. expect(() => definition?.description)
  346. .toThrow(/no run_code schema flavor registered for runtime language "ruby" \(known: "typescript", "python"\)/)
  347. })
  348. it('degrades the run_code flavor to TypeScript when no runtime is mounted (doc-catalog schema harvest)', async () => {
  349. // The tool-catalog generator boots the registry under `mode: code` and
  350. // reads run_code's schema WITHOUT a runtime; peekRuntime returns undefined
  351. // there, so the flavor getter degrades to the TS default rather than
  352. // throwing (that harvest never feeds a model).
  353. const { ctx } = await setup({ mode: 'code', runtime: false })
  354. const definition = ctx.tools.get(RUN_CODE_NAME)
  355. expect(definition?.description).toContain('Execute a TypeScript program')
  356. const params = definition?.parameters as { properties: { code: { description: string } } }
  357. expect(params.properties.code.description).toBe('The program: the body of an async TypeScript function.')
  358. })
  359. it("rejects the assembly when toolOrder names a native tool that mode 'code' no longer contributes", async () => {
  360. const { ctx, systemPrompt } = await setup({ mode: 'code', toolOrder: ['echo', '<unlisted-tools>'] })
  361. registerEcho(ctx)
  362. await expect(systemPrompt.assemble()).rejects.toThrow(/toolOrder lists unregistered tool "echo"/)
  363. })
  364. it('removes run_code and the SDK section when the registry fiber disposes (HMR safety)', async () => {
  365. const ctx = new Context()
  366. await ctx.plugin(SystemPrompt, {})
  367. await ctx.plugin(FakeRuntime, {})
  368. const fiber = await ctx.plugin(ToolRegistry, { mode: 'code' })
  369. expect(ctx.tools.get(RUN_CODE_NAME)).toBeDefined()
  370. await fiber.dispose()
  371. const assembly = await ctx.systemPrompt.assemble()
  372. expect(assembly.tools).toEqual([])
  373. expect(assembly.sections.some(section => section.name === 'tools:sdk')).toBe(false)
  374. })
  375. })
  376. describe('the sub-dispatch scheduler (native concurrency contract)', () => {
  377. /** Register a tool whose calls resolve only when the test releases them; returns live-call telemetry. */
  378. function registerGated(ctx: Context, name: string, concurrencySafe: boolean) {
  379. const gates: (() => void)[] = []
  380. let live = 0
  381. let peak = 0
  382. const order: string[] = []
  383. ctx.tools.register(defineTool({
  384. name,
  385. description: `Gated tool ${name}.`,
  386. parameters: { id: { type: 'string', required: true } },
  387. output: {
  388. schema: { type: 'string' },
  389. render: (_args, value) => [{ type: 'text', text: value }],
  390. },
  391. ...concurrencySafe ? { isConcurrencySafe: () => true } : {},
  392. async execute(args, exec) {
  393. order.push(`start:${args.id}`)
  394. live++
  395. peak = Math.max(peak, live)
  396. // Abort-observing like a real tool: the run-scoped abort releases the
  397. // gate so the bridge's drain reaches quiescence.
  398. await new Promise<void>((release) => {
  399. gates.push(release)
  400. exec.signal.addEventListener('abort', () => { release() }, { once: true })
  401. })
  402. live--
  403. order.push(`end:${args.id}`)
  404. return `${name}:${args.id}`
  405. },
  406. }))
  407. const release = (): void => { gates.shift()?.() }
  408. const releaseAll = (): void => { while (gates.length > 0) gates.shift()!() }
  409. return { order, release, releaseAll, peakLive: () => peak, pending: () => gates.length }
  410. }
  411. it('overlaps concurrency-safe calls under Promise.all and logs a start event per dispatch', async () => {
  412. const { ctx, runtime } = await setup({ mode: 'code' })
  413. const gated = registerGated(ctx, 'safe_read', true)
  414. const { agent, events } = fakeAgent()
  415. runtime.behavior = async (request) => {
  416. const tools = request.bindings[0]!.functions
  417. const all = Promise.all([
  418. tools.safe_read!({ id: 'a' }),
  419. tools.safe_read!({ id: 'b' }),
  420. tools.safe_read!({ id: 'c' }),
  421. ])
  422. // All three must be START-able without any completion (overlap proof).
  423. await expect.poll(() => gated.pending()).toBe(3)
  424. gated.releaseAll()
  425. return { logs: [], value: (await all).map(String).join(',') }
  426. }
  427. const result = await runCode(ctx, 'program', { agent })
  428. expect(result.isError).toBe(false)
  429. expect(gated.peakLive()).toBe(3)
  430. if (result.isError) throw new Error('expected success')
  431. expect(result.value).toMatchObject({ result: 'safe_read:a,safe_read:b,safe_read:c' })
  432. // One start per dispatch, paired with its settle by subCallId, starts in submission order.
  433. const starts = events.filter(event => event.type === 'tool/code-dispatch-start').map(event => event.data as { subCallId: string })
  434. const settles = events.filter(event => event.type === 'tool/code-dispatch').map(event => event.data as { subCallId: string })
  435. expect(starts.map(start => start.subCallId)).toEqual(['call-1:code:1', 'call-1:code:2', 'call-1:code:3'])
  436. expect(new Set(settles.map(settle => settle.subCallId))).toEqual(new Set(starts.map(start => start.subCallId)))
  437. })
  438. it('an exclusive call bars overlap: safe calls drain first, it runs alone, later calls wait', async () => {
  439. const { ctx, runtime } = await setup({ mode: 'code' })
  440. const safe = registerGated(ctx, 'safe_read', true)
  441. const unsafe = registerGated(ctx, 'writer', false)
  442. runtime.behavior = async (request) => {
  443. const tools = request.bindings[0]!.functions
  444. const reads = [tools.safe_read!({ id: 'r1' }), tools.safe_read!({ id: 'r2' })]
  445. const write = tools.writer!({ id: 'w' })
  446. const tail = tools.safe_read!({ id: 'r3' })
  447. await expect.poll(() => safe.pending()).toBe(2)
  448. // The exclusive call must NOT have started while the pool is live.
  449. expect(unsafe.pending()).toBe(0)
  450. safe.releaseAll()
  451. await expect.poll(() => unsafe.pending()).toBe(1)
  452. // The trailing safe call must NOT start while the exclusive one runs.
  453. expect(safe.pending()).toBe(0)
  454. unsafe.release()
  455. await expect.poll(() => safe.pending()).toBe(1)
  456. safe.releaseAll()
  457. await Promise.all([...reads, write, tail])
  458. return { logs: [], value: 'ordered' }
  459. }
  460. const result = await runCode(ctx, 'program')
  461. expect(result.isError).toBe(false)
  462. expect(safe.order.slice(0, 2)).toEqual(['start:r1', 'start:r2'])
  463. expect(unsafe.order).toEqual(['start:w', 'end:w'])
  464. // r3 started only after w ended.
  465. expect(safe.order.indexOf('start:r3')).toBeGreaterThan(safe.order.indexOf('end:r1'))
  466. })
  467. it('maxParallelSubCalls caps the overlap window', async () => {
  468. const { ctx, runtime } = await setup({ mode: 'code', maxParallelSubCalls: 2 })
  469. const gated = registerGated(ctx, 'safe_read', true)
  470. runtime.behavior = async (request) => {
  471. const tools = request.bindings[0]!.functions
  472. const all = Promise.all([
  473. tools.safe_read!({ id: 'a' }),
  474. tools.safe_read!({ id: 'b' }),
  475. tools.safe_read!({ id: 'c' }),
  476. ])
  477. await expect.poll(() => gated.pending()).toBe(2)
  478. // The third call waits for a slot.
  479. expect(gated.pending()).toBe(2)
  480. gated.release()
  481. await expect.poll(() => gated.pending()).toBe(2)
  482. gated.releaseAll()
  483. await all
  484. return { logs: [], value: 'capped' }
  485. }
  486. const result = await runCode(ctx, 'program')
  487. if (result.isError) console.error('CAP-FAIL:', (result.content[0] as { text: string }).text)
  488. expect(result.isError).toBe(false)
  489. expect(gated.peakLive()).toBe(2)
  490. })
  491. it('a tool unregistered between binding enumeration and dispatch fails as unknown tool', async () => {
  492. const { ctx, runtime } = await setup({ mode: 'code' })
  493. const calls: unknown[] = []
  494. const dispose = ctx.tools.register(defineTool({
  495. name: 'ephemeral',
  496. description: 'Unregistered between binding enumeration and dispatch.',
  497. parameters: {},
  498. output: {
  499. schema: { type: 'string' },
  500. render: (_args, value) => [{ type: 'text', text: value }],
  501. },
  502. execute() {
  503. calls.push('ran')
  504. return Promise.resolve('ok')
  505. },
  506. }))
  507. runtime.behavior = async (request) => {
  508. // The binding exists (enumerated at run start); the registry mutation
  509. // makes prepare resolve UNKNOWN_TOOL as a final-result, which commits
  510. // through scheduler.finish (no post-execute).
  511. dispose()
  512. const message = await request.bindings[0]!.functions.ephemeral!({})
  513. .then(() => 'resolved', (error: unknown) => error instanceof Error ? error.message : String(error))
  514. return { logs: [], value: message }
  515. }
  516. const result = await runCode(ctx, 'program')
  517. expect(result.isError).toBe(false)
  518. if (result.isError) throw new Error('expected success')
  519. expect(result.value).toMatchObject({ result: 'unknown tool "ephemeral"' })
  520. expect(calls).toEqual([])
  521. })
  522. it('ordered pre-execute never overlaps: a slow policy on one call delays the next start', async () => {
  523. const { ctx, runtime } = await setup({ mode: 'code' })
  524. const gated = registerGated(ctx, 'safe_read', true)
  525. const stages: string[] = []
  526. let releaseGate: (() => void) | undefined
  527. ctx.on('tools/pre-execute', async (preExec, next) => {
  528. if (preExec.name !== 'safe_read') return next()
  529. stages.push(`pre-enter:${String(preExec.callId)}`)
  530. if (releaseGate === undefined) {
  531. // The FIRST call's policy awaits an asynchronous decision.
  532. await new Promise<void>((resolve) => { releaseGate = resolve })
  533. }
  534. stages.push(`pre-exit:${String(preExec.callId)}`)
  535. return next()
  536. })
  537. runtime.behavior = async (request) => {
  538. const tools = request.bindings[0]!.functions
  539. const all = Promise.all([tools.safe_read!({ id: 'a' }), tools.safe_read!({ id: 'b' })])
  540. // Both submissions are in; the second pre-execute must NOT have entered
  541. // while the first is still awaiting its policy decision.
  542. await expect.poll(() => stages.length).toBeGreaterThanOrEqual(1)
  543. expect(stages).toEqual(['pre-enter:call-1:code:1'])
  544. releaseGate!()
  545. await expect.poll(() => gated.pending()).toBe(2)
  546. gated.releaseAll()
  547. await all
  548. return { logs: [], value: 'ordered-prepare' }
  549. }
  550. const result = await runCode(ctx, 'program')
  551. expect(result.isError).toBe(false)
  552. expect(stages).toEqual([
  553. 'pre-enter:call-1:code:1', 'pre-exit:call-1:code:1',
  554. 'pre-enter:call-1:code:2', 'pre-exit:call-1:code:2',
  555. ])
  556. })
  557. it('an exclusive call holds its barrier through post-execute: the next start waits for the commit', async () => {
  558. const { ctx, runtime } = await setup({ mode: 'code' })
  559. const writer = registerGated(ctx, 'writer', false)
  560. const reader = registerGated(ctx, 'safe_read', true)
  561. const stages: string[] = []
  562. let releasePost: (() => void) | undefined
  563. ctx.on('tools/post-execute', async (postExec, _result, next): Promise<PostToolDecision> => {
  564. if (postExec.name === 'writer') {
  565. stages.push('post-enter:writer')
  566. await new Promise<void>((resolve) => { releasePost = resolve })
  567. stages.push('post-exit:writer')
  568. }
  569. return next()
  570. })
  571. runtime.behavior = async (request) => {
  572. const tools = request.bindings[0]!.functions
  573. const w = tools.writer!({ id: 'w' })
  574. const r = tools.safe_read!({ id: 'r' })
  575. await expect.poll(() => writer.pending()).toBe(1)
  576. writer.release()
  577. // The writer's body is done and its async post-execute is running; the
  578. // parallel read must not have STARTED (no pre/body) while the exclusive
  579. // call's pipeline is still open.
  580. await expect.poll(() => stages).toContain('post-enter:writer')
  581. expect(reader.pending()).toBe(0)
  582. releasePost!()
  583. await w
  584. await expect.poll(() => reader.pending()).toBe(1)
  585. reader.releaseAll()
  586. await r
  587. return { logs: [], value: 'barrier-through-commit' }
  588. }
  589. const result = await runCode(ctx, 'program')
  590. expect(result.isError).toBe(false)
  591. expect(stages).toEqual(['post-enter:writer', 'post-exit:writer'])
  592. })
  593. it('run settlement drains a commit already in progress: the settle event lands inside the turn', async () => {
  594. const { ctx, runtime } = await setup({ mode: 'code' })
  595. const gated = registerGated(ctx, 'safe_read', true)
  596. const { agent, events } = fakeAgent()
  597. let releasePost: (() => void) | undefined
  598. ctx.on('tools/post-execute', async (postExec, _result, next): Promise<PostToolDecision> => {
  599. if (postExec.name === 'safe_read') {
  600. await new Promise<void>((resolve) => { releasePost = resolve })
  601. }
  602. return next()
  603. })
  604. runtime.behavior = async (request) => {
  605. // Fire-and-forget: the program returns while the sub-call's async
  606. // post-execute commit is mid-flight.
  607. request.bindings[0]!.functions.safe_read!({ id: 'a' }).catch(() => 'run-over')
  608. await expect.poll(() => gated.pending()).toBe(1)
  609. gated.release()
  610. await expect.poll(() => releasePost !== undefined).toBe(true)
  611. queueMicrotask(() => { releasePost!() })
  612. return { logs: [], value: 'returned-early' }
  613. }
  614. const result = await runCode(ctx, 'program', { agent })
  615. expect(result.isError).toBe(false)
  616. // The drain awaited the in-progress commit: the settle event exists and
  617. // preceded the run_code turn closing (all appends happen inside
  618. // execute()). The run's settlement aborted the sub-call's signal while
  619. // its post-execute was mid-flight, so the native cancellation contract
  620. // replaces the successful outcome with the aborted result — the event is
  621. // still durable and in-turn, which is the invariant under test.
  622. const settles = events.filter(event => event.type === 'tool/code-dispatch')
  623. expect(settles).toHaveLength(1)
  624. expect(settles[0]?.data).toMatchObject({ name: 'safe_read', isError: true })
  625. })
  626. it('post-execute and context commitment stay in submission order under out-of-order completion', async () => {
  627. const { ctx, runtime } = await setup({ mode: 'code' })
  628. const gated = registerGated(ctx, 'safe_read', true)
  629. const postOrder: string[] = []
  630. ctx.on('tools/post-execute', async (postExec, _result, next): Promise<PostToolDecision> => {
  631. if (postExec.name === 'safe_read') {
  632. postOrder.push(String(postExec.callId))
  633. return {
  634. kind: 'accept' as const,
  635. additionalContexts: [createUserMessage({
  636. content: [{ type: 'text' as const, text: `ctx:${String(postExec.callId)}` }],
  637. source: { kind: 'plugin' as const, plugin: 'order-probe' },
  638. })],
  639. }
  640. }
  641. return next()
  642. })
  643. runtime.behavior = async (request) => {
  644. const tools = request.bindings[0]!.functions
  645. const all = Promise.all([tools.safe_read!({ id: 'a' }), tools.safe_read!({ id: 'b' })])
  646. await expect.poll(() => gated.pending()).toBe(2)
  647. // Complete b FIRST (out of submission order), then a.
  648. gated.release() // releases a (FIFO gate) — invert: release twice reversed is not possible;
  649. gated.releaseAll()
  650. await all
  651. return { logs: [], value: 'ordered-commit' }
  652. }
  653. const result = await runCode(ctx, 'program')
  654. expect(result.isError).toBe(false)
  655. // Post-execute observed submission order regardless of completion interleave.
  656. expect(postOrder).toEqual(['call-1:code:1', 'call-1:code:2'])
  657. // Deferred contexts reach the outer result in the same order.
  658. expect(result.additionalContexts?.map(c => (c.content[0] as { text: string }).text))
  659. .toEqual(['ctx:call-1:code:1', 'ctx:call-1:code:2'])
  660. })
  661. it('a queued-unstarted call abandoned by run settlement logs no start event', async () => {
  662. const { ctx, runtime } = await setup({ mode: 'code' })
  663. const gated = registerGated(ctx, 'writer', false)
  664. const { agent, events } = fakeAgent()
  665. const abandoned: string[] = []
  666. runtime.behavior = async (request) => {
  667. const tools = request.bindings[0]!.functions
  668. // First exclusive call occupies the pool; the second queues unstarted.
  669. // Both rejections are captured (abandonment fires only at settlement,
  670. // AFTER this program has already failed — awaiting it here would deadlock).
  671. tools.writer!({ id: 'w1' }).catch(() => 'settled-under-abort')
  672. tools.writer!({ id: 'w2' }).catch((error: unknown) => {
  673. abandoned.push(error instanceof Error ? error.message : String(error))
  674. })
  675. await expect.poll(() => gated.pending()).toBe(1)
  676. // Fail the program while w1 is in flight and w2 is queued unstarted.
  677. throw new Error('program failed with a queued call')
  678. }
  679. const result = await runCode(ctx, 'program', { agent })
  680. expect(result.isError).toBe(true)
  681. const starts = events.filter(event => event.type === 'tool/code-dispatch-start').map(event => (event.data as { subCallId: string }).subCallId)
  682. const settles = events.filter(event => event.type === 'tool/code-dispatch').map(event => (event.data as { subCallId: string }).subCallId)
  683. // w1 started and settled under the abort; w2 never started and never
  684. // settled — no start event, no settle event, binding rejected with the
  685. // abandonment message at drain time.
  686. expect(starts).toEqual(['call-1:code:1'])
  687. expect(settles).toEqual(['call-1:code:1'])
  688. expect(abandoned).toEqual(['run_code run is over (run_code settled); writer tool call abandoned'])
  689. })
  690. })
  691. describe('the run_code dispatch bridge', () => {
  692. it('bridges tool calls, returns only the curated output, and logs one event per dispatch', async () => {
  693. const { ctx, runtime } = await setup({ mode: 'code' })
  694. const calls = registerEcho(ctx)
  695. const { agent, events } = fakeAgent()
  696. runtime.behavior = async (request) => {
  697. const tools = request.bindings[0]!.functions
  698. const first = await tools.echo!({ value: 'one' })
  699. const second = await tools.echo!({ value: 'two' })
  700. if (typeof first !== 'string' || typeof second !== 'string') throw new Error('echo returned a non-string')
  701. return { logs: [`saw ${first}`], value: second }
  702. }
  703. const result = await runCode(ctx, 'const …: string = …', { agent })
  704. expect(result.isError).toBe(false)
  705. if (result.isError) throw new Error('expected run_code success')
  706. expect(result.value).toEqual({ logs: ['saw echo:one'], result: 'echo:two' })
  707. expect(result.content).toEqual([{ type: 'text', text: 'saw echo:one\necho:two' }])
  708. expect(calls).toEqual([{ value: 'one' }, { value: 'two' }])
  709. const dispatches = events.filter(event => event.type === 'tool/code-dispatch')
  710. expect(dispatches.map(event => event.data)).toEqual([
  711. {
  712. parentCallId: 'call-1', subCallId: 'call-1:code:1', name: 'echo',
  713. arguments: { value: 'one' }, isError: false, content: [{ type: 'text', text: 'echo:one' }],
  714. },
  715. {
  716. parentCallId: 'call-1', subCallId: 'call-1:code:2', name: 'echo',
  717. arguments: { value: 'two' }, isError: false, content: [{ type: 'text', text: 'echo:two' }],
  718. },
  719. ])
  720. expect(result.meta).toBeUndefined()
  721. })
  722. it('exposes only an opaque parent token to nested result observers', async () => {
  723. const { ctx, runtime } = await setup({ mode: 'code' })
  724. registerEcho(ctx)
  725. runtime.behavior = async (request) => {
  726. await request.bindings[0]!.functions.echo!({ value: 'nested' })
  727. return { logs: [], value: 'done' }
  728. }
  729. // Freeze the nested observer's parent correlation. If that were the live
  730. // outer execution object, the timeout-style wrapper could not restore it.
  731. ctx.on('tools/execute', async (exec, next) => {
  732. if (exec.name !== RUN_CODE_NAME) return next()
  733. const previous = exec.signal
  734. exec.signal = new AbortController().signal
  735. const result = await next()
  736. exec.signal = previous
  737. return result
  738. })
  739. ctx.on('tools/result', (exec) => {
  740. if (exec.parent !== undefined) Object.freeze(exec.parent)
  741. })
  742. const result = await runCode(ctx, 'await tools.echo({ value: "nested" })')
  743. expect(result.isError).toBe(false)
  744. expect(result.content).toEqual([{ type: 'text', text: 'done' }])
  745. })
  746. it('forwards a nested terminal conclusion onto the successful run_code result', async () => {
  747. const { ctx, runtime } = await setup({ mode: 'code' })
  748. ctx.tools.register(defineTool({
  749. name: 'finalize',
  750. description: 'Terminal tool.',
  751. parameters: {},
  752. output: {
  753. schema: { type: 'string' },
  754. render: (_args, value) => [{ type: 'text', text: value }],
  755. },
  756. execute(_args, exec) {
  757. exec.concludeTurn()
  758. return Promise.resolve('done')
  759. },
  760. }))
  761. runtime.behavior = async (request) => {
  762. await request.bindings[0]!.functions.finalize!({})
  763. return { logs: [], value: 'program complete' }
  764. }
  765. const concluded = await runCode(ctx, 'await tools.finalize({})')
  766. expect(concluded.isError).toBe(false)
  767. expect(concluded.concludesTurn).toBe(true)
  768. // A policy that converts the nested success into an error strips the
  769. // marker with the result type: the recovering program cannot conclude.
  770. const veto = ctx.on('tools/post-execute', async (exec, _result, next): Promise<PostToolDecision> => {
  771. if (exec.name !== 'finalize') return next()
  772. return { kind: 'block', feedback: [{ type: 'text', text: 'terminal rejected' }] }
  773. })
  774. runtime.behavior = async (request) => {
  775. await request.bindings[0]!.functions.finalize!({}).catch(() => undefined)
  776. return { logs: [], value: 'recovered' }
  777. }
  778. const recovered = await runCode(ctx, 'await tools.finalize({}).catch(() => {})')
  779. veto()
  780. expect(recovered.isError).toBe(false)
  781. expect(recovered.concludesTurn).toBeUndefined()
  782. })
  783. it('serializes Promise.all dispatches: tool executions never overlap, in submission order', async () => {
  784. const { ctx, runtime } = await setup({ mode: 'code' })
  785. const intervals: [string, string][] = []
  786. let active = 0
  787. ctx.tools.register(defineTool({
  788. name: 'probe',
  789. description: 'Records execution overlap.',
  790. parameters: { id: { type: 'string', required: true } },
  791. output: {
  792. schema: { type: 'string' },
  793. render: (_args, value) => [{ type: 'text', text: value }],
  794. },
  795. async execute(args) {
  796. active++
  797. expect(active, 'probe executions overlapped').toBe(1)
  798. intervals.push(['enter', args.id])
  799. await new Promise(resolve => setTimeout(resolve, 20))
  800. intervals.push(['exit', args.id])
  801. active--
  802. return args.id
  803. },
  804. }))
  805. runtime.behavior = async (request) => {
  806. const tools = request.bindings[0]!.functions
  807. const values = await Promise.all([tools.probe!({ id: 'a' }), tools.probe!({ id: 'b' }), tools.probe!({ id: 'c' })])
  808. if (!values.every(value => typeof value === 'string')) throw new Error('probe returned a non-string')
  809. return { logs: [], value: values.join(',') }
  810. }
  811. const result = await runCode(ctx, 'program')
  812. expect(result.isError).toBe(false)
  813. expect(intervals).toEqual([
  814. ['enter', 'a'], ['exit', 'a'],
  815. ['enter', 'b'], ['exit', 'b'],
  816. ['enter', 'c'], ['exit', 'c'],
  817. ])
  818. expect(result.content[0]).toEqual({ type: 'text', text: 'a,b,c' })
  819. })
  820. it('rejects the program-side call when the tool errors, with the tool error text', async () => {
  821. const { ctx, runtime } = await setup({ mode: 'code' })
  822. ctx.tools.register(defineContentToolFixture({
  823. name: 'fail',
  824. description: 'Always fails.',
  825. parameters: {},
  826. execute(): Promise<never> { return Promise.reject(new Error('deliberate failure')) },
  827. }))
  828. runtime.behavior = async (request) => {
  829. try {
  830. await request.bindings[0]!.functions.fail!({})
  831. return { logs: [], value: 'unreachable' }
  832. } catch (error: unknown) {
  833. return { logs: [], value: `caught: ${error instanceof Error ? error.message : String(error)}` }
  834. }
  835. }
  836. const result = await runCode(ctx, 'program')
  837. expect(result.content[0]).toEqual({ type: 'text', text: 'caught: deliberate failure' })
  838. })
  839. it('a throwing tools/code-dispatch-log listener is contained: the unshaped content is logged', async () => {
  840. const { ctx, runtime } = await setup({ mode: 'code' })
  841. registerEcho(ctx)
  842. ctx.on('tools/code-dispatch-log', () => { throw new Error('shaper exploded') })
  843. const { agent, events } = fakeAgent()
  844. runtime.behavior = async (request) => {
  845. const value = await request.bindings[0]!.functions.echo!({ value: 'x' })
  846. return { logs: [], value: value as string }
  847. }
  848. const result = await runCode(ctx, 'program', { agent })
  849. expect(result.isError).toBe(false)
  850. const settle = events.find(event => event.type === 'tool/code-dispatch')
  851. expect(settle?.data).toMatchObject({ name: 'echo', isError: false, content: [{ type: 'text', text: 'echo:x' }] })
  852. })
  853. it('a throwing tools/pre-execute listener settles the sub-call without post-execute', async () => {
  854. const { ctx, runtime } = await setup({ mode: 'code' })
  855. const calls = registerEcho(ctx)
  856. const postExecuted: string[] = []
  857. ctx.on('tools/pre-execute', (exec, next) => {
  858. if (exec.name === 'echo') throw new Error('gate exploded')
  859. return next()
  860. })
  861. ctx.on('tools/post-execute', (exec, _result, next): Promise<PostToolDecision> => {
  862. if (exec.name === 'echo') postExecuted.push(exec.name)
  863. return next()
  864. })
  865. const { agent, events } = fakeAgent()
  866. runtime.behavior = async (request) => {
  867. const message = await request.bindings[0]!.functions.echo!({ value: 'x' })
  868. .then(() => 'resolved', (error: unknown) => error instanceof Error ? error.message : String(error))
  869. return { logs: [], value: message }
  870. }
  871. const result = await runCode(ctx, 'program', { agent })
  872. expect(result.isError).toBe(false)
  873. if (result.isError) throw new Error('expected success')
  874. expect(result.value).toMatchObject({ result: 'gate exploded' })
  875. // The pipeline failure is final: the body never ran and post-execute was
  876. // skipped, yet the settle event still carries the error outcome.
  877. expect(calls).toEqual([])
  878. expect(postExecuted).toEqual([])
  879. const settles = events.filter(event => event.type === 'tool/code-dispatch')
  880. expect(settles).toHaveLength(1)
  881. expect(settles[0]?.data).toMatchObject({ name: 'echo', isError: true })
  882. })
  883. it('a tools/pre-execute deny reaches the program as a binding rejection', async () => {
  884. const { ctx, runtime } = await setup({ mode: 'code' })
  885. registerEcho(ctx)
  886. ctx.on('tools/pre-execute', (exec, next) => {
  887. if (exec.name === 'echo') return Promise.resolve({ kind: 'deny' as const, reason: 'not on my watch' })
  888. return next()
  889. })
  890. runtime.behavior = async (request) => {
  891. try {
  892. await request.bindings[0]!.functions.echo!({ value: 'x' })
  893. return { logs: [], value: 'unreachable' }
  894. } catch (error: unknown) {
  895. return { logs: [], value: `denied: ${error instanceof Error ? error.message : String(error)}` }
  896. }
  897. }
  898. const result = await runCode(ctx, 'program')
  899. expect(result.content[0]?.type).toBe('text')
  900. expect((result.content[0] as { text: string }).text).toContain('not on my watch')
  901. })
  902. it('rejects a binding argument that is not lossless JSON, dispatching nothing', async () => {
  903. const { ctx, runtime } = await setup({ mode: 'code' })
  904. const calls = registerEcho(ctx)
  905. const { agent, events } = fakeAgent()
  906. runtime.behavior = async (request) => {
  907. try {
  908. await request.bindings[0]!.functions.echo!({ value: 'x', big: 1n })
  909. return { logs: [], value: 'unreachable' }
  910. } catch (error: unknown) {
  911. return { logs: [], value: error instanceof Error ? error.message : String(error) }
  912. }
  913. }
  914. const result = await runCode(ctx, 'program', { agent })
  915. expect((result.content[0] as { text: string }).text).toContain('lossless JSON')
  916. expect(calls).toEqual([])
  917. expect(events.filter(event => event.type === 'tool/code-dispatch')).toEqual([])
  918. })
  919. it('dispatches and logs independent snapshots of the same lossless JSON value', async () => {
  920. const { ctx, runtime } = await setup({ mode: 'code' })
  921. const calls = registerEcho(ctx)
  922. const { agent, events } = fakeAgent()
  923. runtime.behavior = async (request) => {
  924. const args = Object.assign(Object.create(null) as Record<string, unknown>, { value: 'x', nested: ['same'] })
  925. await request.bindings[0]!.functions.echo!(args)
  926. return { logs: [] }
  927. }
  928. await runCode(ctx, 'program', { agent })
  929. expect(calls).toEqual([{ value: 'x', nested: ['same'] }])
  930. const dispatch = events.find(event => event.type === 'tool/code-dispatch')?.data as SessionEventMap['tool/code-dispatch']
  931. expect(dispatch.arguments).toEqual({ value: 'x', nested: ['same'] })
  932. })
  933. it('defers sub-call additionalContexts onto the outer run_code result', async () => {
  934. const { ctx, runtime } = await setup({ mode: 'code' })
  935. registerEcho(ctx)
  936. ctx.on('tools/post-execute', (exec, _result, next): Promise<PostToolDecision> => {
  937. if (exec.name === 'echo') {
  938. return Promise.resolve({
  939. kind: 'accept' as const,
  940. additionalContexts: [createUserMessage({
  941. content: [{ type: 'text' as const, text: `context for ${exec.callId}` }],
  942. source: { kind: 'plugin' as const, plugin: 'test' },
  943. })],
  944. })
  945. }
  946. return next()
  947. })
  948. runtime.behavior = async (request) => {
  949. await request.bindings[0]!.functions.echo!({ value: 'x' })
  950. await request.bindings[0]!.functions.echo!({ value: 'y' })
  951. return { logs: [], value: 'done' }
  952. }
  953. const result = await runCode(ctx, 'program')
  954. expect(result.isError).toBe(false)
  955. expect(result.additionalContexts).toMatchObject([
  956. {
  957. role: 'user',
  958. content: [{ type: 'text', text: 'context for call-1:code:1' }],
  959. source: { kind: 'plugin', plugin: 'test' },
  960. },
  961. {
  962. role: 'user',
  963. content: [{ type: 'text', text: 'context for call-1:code:2' }],
  964. source: { kind: 'plugin', plugin: 'test' },
  965. },
  966. ])
  967. })
  968. it('keeps sub-call contexts when run_code fails after the nested dispatch', async () => {
  969. const { ctx, runtime } = await setup({ mode: 'both' })
  970. registerEcho(ctx)
  971. ctx.on('tools/post-execute', (exec, _result, next): Promise<PostToolDecision> => {
  972. if (exec.name !== 'echo') return next()
  973. return Promise.resolve({
  974. kind: 'accept',
  975. additionalContexts: [createUserMessage({
  976. content: [{ type: 'text', text: 'nested context' }],
  977. source: { kind: 'plugin', plugin: 'test' },
  978. })],
  979. })
  980. })
  981. runtime.behavior = async (request) => {
  982. await request.bindings[0]!.functions.echo!({ value: 'x' })
  983. return { logs: [], error: { kind: 'exception', message: 'program failed later' } }
  984. }
  985. const result = await runCode(ctx, 'program')
  986. expect(result.isError).toBe(true)
  987. expect(result.additionalContexts).toEqual([{
  988. id: expect.any(String) as unknown,
  989. role: 'user',
  990. content: [{ type: 'text', text: 'nested context' }],
  991. source: { kind: 'plugin', plugin: 'test' },
  992. }])
  993. })
  994. it('converts a failed run into a structured isError result carrying kind, message, and captured logs', async () => {
  995. const { ctx, runtime } = await setup({ mode: 'code' })
  996. runtime.behavior = () => Promise.resolve({
  997. logs: ['got this far'],
  998. error: { kind: 'timeout', message: 'compute budget exhausted (300ms busy)' },
  999. })
  1000. const result = await runCode(ctx, 'program')
  1001. expect(result.isError).toBe(true)
  1002. expect(result.error).toMatchObject({ info: { name: 'CodeRunFailedError', code: 'CODE_RUN_FAILED' } })
  1003. const text = (result.content[0] as { text: string }).text
  1004. expect(text).toContain('code run failed (timeout)')
  1005. expect(text).toContain('compute budget exhausted')
  1006. expect(text).toContain('got this far')
  1007. })
  1008. it('CodeRunFailedError is a HarnessError with the CODE_RUN_FAILED code', () => {
  1009. const error = new CodeRunFailedError('boom')
  1010. expect(error.code).toBe('CODE_RUN_FAILED')
  1011. expect(error.name).toBe('CodeRunFailedError')
  1012. })
  1013. it('aborting the outer signal aborts the in-flight sub-dispatch and abandons queued ones', async () => {
  1014. const { ctx, runtime } = await setup({ mode: 'code' })
  1015. const seen: string[] = []
  1016. let sawAbort = false
  1017. ctx.tools.register(defineContentToolFixture({
  1018. name: 'slow',
  1019. description: 'Slow tool observing its signal.',
  1020. parameters: { id: { type: 'string', required: true } },
  1021. async execute(args, exec) {
  1022. seen.push(args.id)
  1023. await new Promise<void>((resolve) => {
  1024. const timer = setTimeout(resolve, 500)
  1025. exec.signal.addEventListener('abort', () => { sawAbort = true; clearTimeout(timer); resolve() }, { once: true })
  1026. })
  1027. return [{ type: 'text' as const, text: args.id }]
  1028. },
  1029. }))
  1030. const controller = new AbortController()
  1031. runtime.behavior = async (request) => {
  1032. const tools = request.bindings[0]!.functions
  1033. const calls = [tools.slow!({ id: 'first' }).catch(() => 'rejected'), tools.slow!({ id: 'second' }).catch(() => 'rejected')]
  1034. setTimeout(() => { controller.abort('user-cancel') }, 50)
  1035. await Promise.all(calls)
  1036. // A real runtime would be terminated by the abort; the fake honors the
  1037. // contract by reporting the abort as the run failure.
  1038. return { logs: [], error: { kind: 'abort', message: 'user-cancel' } }
  1039. }
  1040. const result = await runCode(ctx, 'program', { signal: controller.signal })
  1041. expect(result.isError).toBe(true)
  1042. expect((result.content[0] as { text: string }).text).toContain('code run failed (abort)')
  1043. expect(seen).toEqual(['first'])
  1044. expect(sawAbort).toBe(true)
  1045. })
  1046. it('a runtime that starts a binding call and then REJECTS still reaches quiescence before returning', async () => {
  1047. const { ctx, runtime } = await setup({ mode: 'code' })
  1048. const { agent, events } = fakeAgent()
  1049. let sawAbort = false
  1050. let started!: () => void
  1051. const inFlight = new Promise<void>((resolve) => { started = resolve })
  1052. ctx.tools.register(defineContentToolFixture({
  1053. name: 'slow',
  1054. description: 'Slow tool observing its signal.',
  1055. parameters: { id: { type: 'string', required: true } },
  1056. async execute(args, exec) {
  1057. started()
  1058. await new Promise<void>((resolve) => {
  1059. const timer = setTimeout(resolve, 500)
  1060. exec.signal.addEventListener('abort', () => { sawAbort = true; clearTimeout(timer); resolve() }, { once: true })
  1061. })
  1062. return [{ type: 'text' as const, text: args.id }]
  1063. },
  1064. }))
  1065. runtime.behavior = async (request) => {
  1066. // Start a sub-dispatch, keep its rejection held, and fail the run once the tool is
  1067. // genuinely in flight — a seam error after work has begun.
  1068. request.bindings[0]!.functions.slow!({ id: 'orphan' }).catch(() => 'held')
  1069. await inFlight
  1070. throw new Error('backend exploded')
  1071. }
  1072. const result = await runCode(ctx, 'program', { agent })
  1073. expect(result.isError).toBe(true)
  1074. expect((result.content[0] as { text: string }).text).toContain('backend exploded')
  1075. // Quiescence held: the in-flight sub-dispatch was aborted and its event
  1076. // logged INSIDE the run_code execution, not after it returned.
  1077. expect(sawAbort).toBe(true)
  1078. expect(events.filter(event => event.type === 'tool/code-dispatch').map(event => (event.data as { name: string }).name)).toEqual(['slow'])
  1079. })
  1080. it('runs without an owning agent: dispatches work, event logging is skipped', async () => {
  1081. const { ctx, runtime } = await setup({ mode: 'code' })
  1082. const calls = registerEcho(ctx)
  1083. runtime.behavior = async (request) => {
  1084. await request.bindings[0]!.functions.echo!({ value: 'x' })
  1085. return { logs: [], value: 'ok' }
  1086. }
  1087. const result = await runCode(ctx, 'program')
  1088. expect(result.isError).toBe(false)
  1089. expect(calls).toEqual([{ value: 'x' }])
  1090. })
  1091. it('executing run_code under a missing runtime is a structured isError, not a crash', async () => {
  1092. const ctx = new Context()
  1093. await ctx.plugin(SystemPrompt, {})
  1094. await ctx.plugin(ToolRegistry, { mode: 'code' })
  1095. const result = await runCode(ctx, 'program')
  1096. expect(result.isError).toBe(true)
  1097. expect((result.content[0] as { text: string }).text).toContain('requires a code runtime')
  1098. })
  1099. it('presents the model-authored description as the execute-card title over the program input', async () => {
  1100. const { ctx } = await setup({ mode: 'code' })
  1101. const tool = ctx.tools.get(RUN_CODE_NAME)!
  1102. // The description labels the card (the bash description precedent); the
  1103. // program itself remains the expanded raw input.
  1104. expect(tool.presentCall?.({ code: 'return 1', description: 'Return the constant one' })).toEqual({
  1105. card: 'generic',
  1106. title: 'Return the constant one',
  1107. kind: 'execute',
  1108. rawInput: 'return 1',
  1109. })
  1110. })
  1111. it('rejects a whitespace-only description with a structured isError', async () => {
  1112. const { ctx } = await setup({ mode: 'code' })
  1113. const result = await runCode(ctx, 'return 1', { description: ' ' })
  1114. expect(result.isError).toBe(true)
  1115. expect((result.content[0] as { text: string }).text).toContain('invalid description')
  1116. })
  1117. it.each([
  1118. ['logs only', { logs: ['printed'] }, 'printed'],
  1119. ['result only', { logs: [], value: 'returned' }, 'returned'],
  1120. ['logs plus result', { logs: ['printed'], value: 'returned' }, 'printed\nreturned'],
  1121. ['no output', { logs: [] }, '(run_code completed with no output)'],
  1122. ] as [string, CodeRunResult, string][])('keeps %s in durable content without a result presenter', async (_name, output, text) => {
  1123. const { ctx, runtime } = await setup({ mode: 'code' })
  1124. runtime.behavior = () => Promise.resolve(output)
  1125. const result = await runCode(ctx, 'return 1')
  1126. const tool = ctx.tools.get(RUN_CODE_NAME)!
  1127. expect(result.content).toEqual([{ type: 'text', text }])
  1128. // Surfaces keep the pending program title and render this durable content
  1129. // through their generic fallback. Omitting a result view also prevents the
  1130. // host frame from carrying the same raw content a second time.
  1131. expect('presentResult' in tool).toBe(false)
  1132. })
  1133. it('keeps a post-policy spill preview in durable content without a result presenter', async () => {
  1134. const { ctx, runtime } = await setup({ mode: 'code' })
  1135. const preview = 'HEAD\n\n(Omitted 100 bytes. Full formatted result stored at: /tmp/run-code.txt.)\n\nTAIL'
  1136. runtime.behavior = () => Promise.resolve({ logs: ['printed'], value: 'returned' })
  1137. ctx.on('tools/post-execute', (exec, _result, next): Promise<PostToolDecision> => {
  1138. if (exec.name !== RUN_CODE_NAME) return next()
  1139. return Promise.resolve({ kind: 'accept', content: [{ type: 'text', text: preview }] })
  1140. })
  1141. const result = await runCode(ctx, 'return 1')
  1142. const tool = ctx.tools.get(RUN_CODE_NAME)!
  1143. expect(result.content).toEqual([{ type: 'text', text: preview }])
  1144. expect('presentResult' in tool).toBe(false)
  1145. })
  1146. it('keeps canonical failure content durable without a result presenter', async () => {
  1147. const { ctx, runtime } = await setup({ mode: 'code' })
  1148. runtime.behavior = () => Promise.resolve({
  1149. logs: ['captured before failure'],
  1150. error: { kind: 'output-limit', message: 'outer output exceeded 8 bytes' },
  1151. })
  1152. const result = await runCode(ctx, 'return 1')
  1153. const tool = ctx.tools.get(RUN_CODE_NAME)!
  1154. expect(result.isError).toBe(true)
  1155. expect(result.content).toEqual([{
  1156. type: 'text',
  1157. text: 'Error: code run failed (output-limit): outer output exceeded 8 bytes\nCaptured output:\ncaptured before failure',
  1158. }])
  1159. expect('presentResult' in tool).toBe(false)
  1160. })
  1161. it('logs the complete sub-result content verbatim, non-text blocks and long text included', async () => {
  1162. const { ctx, runtime } = await setup({ mode: 'code' })
  1163. const { agent, events } = fakeAgent()
  1164. const long = 'x'.repeat(300)
  1165. ctx.tools.register(defineTool({
  1166. name: 'mixed',
  1167. description: 'Returns mixed content.',
  1168. parameters: {},
  1169. output: {
  1170. schema: { type: 'string' },
  1171. render: () => [
  1172. { type: 'text', text: long },
  1173. { type: 'reasoning', text: 'hidden' },
  1174. ],
  1175. },
  1176. execute() {
  1177. return Promise.resolve('mixed-value')
  1178. },
  1179. }))
  1180. runtime.behavior = async (request) => {
  1181. const value = await request.bindings[0]!.functions.mixed!({})
  1182. return { logs: [], value }
  1183. }
  1184. const result = await runCode(ctx, 'program', { agent })
  1185. expect(result.isError).toBe(false)
  1186. expect((result.content[0] as { text: string }).text).toBe('mixed-value')
  1187. const dispatch = events.find(event => event.type === 'tool/code-dispatch')?.data as SessionEventMap['tool/code-dispatch']
  1188. expect(dispatch.content).toEqual([
  1189. { type: 'text', text: long },
  1190. { type: 'reasoning', text: 'hidden' },
  1191. ])
  1192. })
  1193. it('rejects undefined, getter-throwing, exotic, and unrepresentable binding arguments before dispatch', async () => {
  1194. const { ctx, runtime } = await setup({ mode: 'code' })
  1195. const calls = registerEcho(ctx)
  1196. const { agent, events } = fakeAgent()
  1197. runtime.behavior = async (request) => {
  1198. const echo = request.bindings[0]!.functions.echo!
  1199. const catchMessage = (promise: Promise<unknown>) => promise.then(() => 'resolved', (error: unknown) => error instanceof Error ? error.message : String(error))
  1200. return {
  1201. logs: [],
  1202. value: [
  1203. // Root undefined must reject up front: the event log rejects it as
  1204. // data, and nothing may execute unlogged.
  1205. await catchMessage(echo(undefined)),
  1206. await catchMessage(echo(Object.defineProperty({}, 'bad', { enumerable: true, get() { throw 'raw-throw' } }))),
  1207. await catchMessage(echo(Object.defineProperty({}, 'bad', { enumerable: true, get() { throw new Error('error-throw') } }))),
  1208. await catchMessage(echo(new Date(0))),
  1209. // A bare function is a value JSON cannot represent at all.
  1210. await catchMessage(echo(() => 1)),
  1211. ].join(' | '),
  1212. }
  1213. }
  1214. const result = await runCode(ctx, 'program', { agent })
  1215. const text = (result.content[0] as { text: string }).text
  1216. expect(text).toContain('call the tool with an arguments object')
  1217. expect(text).toContain('lossless JSON: raw-throw')
  1218. expect(text).toContain('lossless JSON: error-throw')
  1219. expect(text.match(/tool arguments must be lossless JSON/g)).toHaveLength(5)
  1220. // None dispatched or logged.
  1221. expect(calls).toEqual([])
  1222. expect(events.filter(event => event.type === 'tool/code-dispatch')).toEqual([])
  1223. })
  1224. it('dispatches and durably logs binding arguments deeper than the structured-clone call stack', async () => {
  1225. const { ctx, runtime } = await setup({ mode: 'code' })
  1226. const depth = 5_000
  1227. let observedDepth = 0
  1228. let observedLeaf: JsonValue | undefined
  1229. ctx.tools.register(defineTool({
  1230. name: 'deep_args',
  1231. description: 'Measure a deeply nested JSON argument.',
  1232. parameters: { nested: { type: 'json', required: true } },
  1233. output: {
  1234. schema: { type: 'integer' },
  1235. render: (_args, value) => [{ type: 'text', text: String(value) }],
  1236. },
  1237. execute(args) {
  1238. let cursor = args.nested
  1239. while (Array.isArray(cursor)) {
  1240. if (cursor.length !== 1) throw new Error('expected one item per nesting layer')
  1241. observedDepth++
  1242. cursor = cursor[0]!
  1243. }
  1244. observedLeaf = cursor
  1245. return Promise.resolve(observedDepth)
  1246. },
  1247. }))
  1248. const session = Session.create(SessionId('deep-code-arguments'))
  1249. const agent = { session } as Agent
  1250. runtime.behavior = async (request) => {
  1251. let nested: JsonValue = 'leaf'
  1252. for (let index = 0; index < depth; index++) nested = [nested]
  1253. const value = await request.bindings[0]!.functions.deep_args!({ nested })
  1254. return { logs: [], value }
  1255. }
  1256. const result = await runCode(ctx, 'return tools.deep_args(...)', { agent })
  1257. expect(result.isError).toBe(false)
  1258. expect(result.isError ? undefined : result.value).toEqual({ logs: [], result: depth })
  1259. expect({ observedDepth, observedLeaf }).toEqual({ observedDepth: depth, observedLeaf: 'leaf' })
  1260. const dispatch = session.events.find(event => event.type === 'tool/code-dispatch')
  1261. if (dispatch === undefined) throw new Error('expected a durable tool/code-dispatch event')
  1262. const logged = dispatch.data.arguments as { nested: JsonValue }
  1263. let loggedDepth = 0
  1264. let loggedCursor = logged.nested
  1265. while (Array.isArray(loggedCursor)) {
  1266. if (loggedCursor.length !== 1) throw new Error('expected one logged item per nesting layer')
  1267. loggedDepth++
  1268. loggedCursor = loggedCursor[0]!
  1269. }
  1270. expect({ loggedDepth, loggedCursor }).toEqual({ loggedDepth: depth, loggedCursor: 'leaf' })
  1271. })
  1272. it('gives the tool and durable log the same immutable argument value', async () => {
  1273. const { ctx, runtime } = await setup({ mode: 'code' })
  1274. const { agent, events } = fakeAgent()
  1275. let mutationSucceeded: boolean | undefined
  1276. ctx.tools.register(defineContentToolFixture({
  1277. name: 'mutator',
  1278. description: 'Attempts to mutate its args object.',
  1279. parameters: { list: { type: 'array', required: true } },
  1280. execute(args) {
  1281. mutationSucceeded = Reflect.set(args.list, 1, 'injected-by-tool')
  1282. return Promise.resolve([{ type: 'text' as const, text: 'protected' }])
  1283. },
  1284. }))
  1285. runtime.behavior = async (request) => {
  1286. await request.bindings[0]!.functions.mutator!({ list: ['original'] })
  1287. return { logs: [] }
  1288. }
  1289. const result = await runCode(ctx, 'program', { agent })
  1290. expect(result.isError).toBe(false)
  1291. expect(mutationSucceeded).toBe(false)
  1292. const dispatch = events.find(event => event.type === 'tool/code-dispatch')?.data as SessionEventMap['tool/code-dispatch']
  1293. expect(dispatch.arguments).toEqual({ list: ['original'] })
  1294. })
  1295. it('exposes a tool named __proto__ as an ordinary own binding', async () => {
  1296. const { ctx, runtime } = await setup({ mode: 'code' })
  1297. ctx.tools.register(defineTool({
  1298. name: '__proto__',
  1299. description: 'A prototype-colliding tool name.',
  1300. parameters: {},
  1301. output: {
  1302. schema: { type: 'string' },
  1303. render: (_args, value) => [{ type: 'text', text: value }],
  1304. },
  1305. execute() { return Promise.resolve('proto-tool-ok') },
  1306. }))
  1307. runtime.behavior = async (request) => {
  1308. const functions = request.bindings[0]!.functions
  1309. expect(Object.getPrototypeOf(functions)).toBeNull()
  1310. const value = await functions['__proto__']!({})
  1311. return { logs: [], value }
  1312. }
  1313. const result = await runCode(ctx, 'program')
  1314. expect(result.isError).toBe(false)
  1315. expect(result.content[0]).toEqual({ type: 'text', text: 'proto-tool-ok' })
  1316. })
  1317. it('renders every non-string JSON root as pretty JSON while preserving strings raw', async () => {
  1318. const { ctx, runtime } = await setup({ mode: 'code' })
  1319. runtime.behavior = () => Promise.resolve({ logs: [], value: { n: 42, ok: true } })
  1320. expect((await runCode(ctx, 'object')).content[0]).toEqual({ type: 'text', text: '{\n "n": 42,\n "ok": true\n}' })
  1321. runtime.behavior = () => Promise.resolve({ logs: [], value: {} })
  1322. expect((await runCode(ctx, 'empty object')).content[0]).toEqual({ type: 'text', text: '{}' })
  1323. const nested = { outer: [{ inner: true }] }
  1324. runtime.behavior = () => Promise.resolve({ logs: [], value: nested })
  1325. expect((await runCode(ctx, 'nested')).content[0]).toEqual({ type: 'text', text: JSON.stringify(nested, null, 2) })
  1326. runtime.behavior = () => Promise.resolve({ logs: [], value: ['x', 2] })
  1327. expect((await runCode(ctx, 'array')).content[0]).toEqual({ type: 'text', text: '[\n "x",\n 2\n]' })
  1328. runtime.behavior = () => Promise.resolve({ logs: [], value: [] })
  1329. expect((await runCode(ctx, 'empty array')).content[0]).toEqual({ type: 'text', text: '[]' })
  1330. runtime.behavior = () => Promise.resolve({ logs: [], value: null })
  1331. expect((await runCode(ctx, 'null')).content[0]).toEqual({ type: 'text', text: 'null' })
  1332. runtime.behavior = () => Promise.resolve({ logs: [], value: 'raw' })
  1333. expect((await runCode(ctx, 'string')).content[0]).toEqual({ type: 'text', text: 'raw' })
  1334. runtime.behavior = () => Promise.resolve({ logs: [] })
  1335. const absent = await runCode(ctx, 'undefined')
  1336. expect(absent.content[0]).toEqual({ type: 'text', text: '(run_code completed with no output)' })
  1337. expect(absent.isError ? undefined : absent.value).toEqual({ logs: [] })
  1338. })
  1339. it('renders deeply nested JSON without recursive traversal or quadratic indentation', async () => {
  1340. const { ctx, runtime } = await setup({ mode: 'code' })
  1341. let value: JsonValue = {
  1342. emptyArray: [],
  1343. emptyObject: {},
  1344. pair: ['leaf', 2],
  1345. record: { first: true, second: null },
  1346. }
  1347. for (let depth = 0; depth < 5_000; depth++) value = [value]
  1348. runtime.behavior = () => Promise.resolve({ logs: [], value })
  1349. const result = await runCode(ctx, 'deep result')
  1350. expect(result.isError).toBe(false)
  1351. const text = (result.content[0] as { type: 'text'; text: string }).text
  1352. expect(text.startsWith('[\n [\n [')).toBe(true)
  1353. expect(text).toContain('"leaf"')
  1354. expect(text.endsWith(']')).toBe(true)
  1355. expect(text.length).toBeLessThan(11_000)
  1356. })
  1357. it('short-circuits a pre-aborted outer signal before the code runtime', async () => {
  1358. const { ctx, runtime } = await setup({ mode: 'code' })
  1359. const calls = registerEcho(ctx)
  1360. runtime.behavior = (request) => {
  1361. // The fake honors the seam contract for an already-aborted signal.
  1362. if (request.signal?.aborted) return Promise.resolve({ logs: [], error: { kind: 'abort' as const, message: String(request.signal.reason) } })
  1363. return Promise.resolve({ logs: [], value: 'unreachable' })
  1364. }
  1365. const controller = new AbortController()
  1366. controller.abort('too-late')
  1367. const result = await runCode(ctx, 'program', { signal: controller.signal })
  1368. expect(result.isError).toBe(true)
  1369. expect(result).toEqual({
  1370. content: [{ type: 'text', text: 'Error: tool call aborted before dispatch' }],
  1371. isError: true,
  1372. error: {
  1373. message: 'tool call aborted before dispatch',
  1374. info: { name: 'AbortError', code: TOOL_ABORTED_BEFORE_DISPATCH },
  1375. },
  1376. })
  1377. expect(runtime.lastRequest).toBeUndefined()
  1378. expect(calls).toEqual([])
  1379. })
  1380. it('reports cancellation after rejecting a late binding without dispatching it', async () => {
  1381. const { ctx, runtime } = await setup({ mode: 'code' })
  1382. const calls = registerEcho(ctx)
  1383. const controller = new AbortController()
  1384. runtime.behavior = async (request) => {
  1385. controller.abort('cancelled-mid-run')
  1386. const message = await request.bindings[0]!.functions.echo!({ value: 'x' })
  1387. .then(() => 'resolved', (error: unknown) => error instanceof Error ? error.message : String(error))
  1388. return { logs: [], value: message }
  1389. }
  1390. const result = await runCode(ctx, 'program', { signal: controller.signal })
  1391. expect(result.isError).toBe(true)
  1392. expect(result.error).toEqual({
  1393. message: 'tool call aborted',
  1394. info: { name: 'AbortError', code: 'ABORTED' },
  1395. })
  1396. expect((result.content[0] as { text: string }).text).toBe('Error: tool call aborted')
  1397. expect(calls).toEqual([])
  1398. })
  1399. it('a tool/code-dispatch event never derives a model message', () => {
  1400. const session = Session.create(SessionId('code-mode-derive'))
  1401. session.append('user/message', createUserMessage({
  1402. content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' },
  1403. }), { surfaceOp: 'append' })
  1404. session.append('tool/code-dispatch', {
  1405. parentCallId: CallId('p1'),
  1406. subCallId: CallId('p1:code:1'),
  1407. name: 'echo',
  1408. arguments: { value: 'x' },
  1409. isError: false,
  1410. content: [{ type: 'text', text: 'echo:x' }],
  1411. })
  1412. const derived = session.deriveMessages()
  1413. expect(derived).toHaveLength(1)
  1414. expect(derived[0]?.role).toBe('user')
  1415. })
  1416. it('direct construction rejects a non-positive parallel sub-call cap at load', async () => {
  1417. const ctx = new Context()
  1418. await ctx.plugin(SystemPrompt, {})
  1419. expect(() => new ToolRegistry(ctx, { mode: 'code', maxParallelSubCalls: 0 }))
  1420. .toThrow('maxParallelSubCalls must be a positive integer')
  1421. })
  1422. it('direct construction in code mode defaults the parallel sub-call cap', async () => {
  1423. const ctx = new Context()
  1424. await ctx.plugin(SystemPrompt, {})
  1425. const registry = new ToolRegistry(ctx, { mode: 'code' })
  1426. expect(registry.get(RUN_CODE_NAME)).toBeDefined()
  1427. })
  1428. it('defaults to native mode under direct construction with no config', async () => {
  1429. const ctx = new Context()
  1430. await ctx.plugin(SystemPrompt, {})
  1431. const registry = new ToolRegistry(ctx)
  1432. expect(registry.get(RUN_CODE_NAME)).toBeUndefined()
  1433. const assembly = await ctx.systemPrompt.assemble()
  1434. expect(assembly.sections.some(section => section.name === 'tools:sdk')).toBe(false)
  1435. })
  1436. })