structured.spec.ts 36 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789
  1. import { describe, expect, it } from 'vitest'
  2. import { Context } from 'cordis'
  3. import LlmService, { CallId, type ContentBlock, type GenerateOptions } from '@deepseek-ai/dsh-llm'
  4. import SessionStore from '@deepseek-ai/dsh-session'
  5. import SystemPrompt from '@deepseek-ai/dsh-system-prompt'
  6. import ToolRegistry from '@deepseek-ai/dsh-tools'
  7. import AgentRegistry, { AgentId } from '@deepseek-ai/dsh-agent'
  8. import type { ContinuationDecision } from '@deepseek-ai/dsh-agent'
  9. import AgentLoop from '@deepseek-ai/dsh-agent-loop'
  10. import * as Invariants from '@deepseek-ai/dsh-invariants'
  11. import SubagentService, { type SubagentStartRequest } from '@deepseek-ai/dsh-subagent'
  12. import type { Config as ToolConfig, StructuredOutputSchema } from '@deepseek-ai/dsh-tools'
  13. import { RUN_CODE_NAME } from '@deepseek-ai/dsh-tools'
  14. import { MockAdapter, textResponse, toolCallResponse } from '../../../core/agent-loop/tests/mock-adapter.ts'
  15. import { startInProcessRun } from '../src/index.ts'
  16. import {
  17. STRUCTURED_OUTPUT_INSTRUCTION,
  18. STRUCTURED_OUTPUT_TOOL,
  19. } from '../src/structured.ts'
  20. type Script = ConstructorParameters<typeof MockAdapter>[0]
  21. interface CodeRunRequestLike {
  22. bindings: { global: string; functions: Record<string, (args: unknown) => Promise<unknown>> }[]
  23. }
  24. interface SetupOptions {
  25. toolMode?: ToolConfig['mode']
  26. codeRun?: (request: CodeRunRequestLike) => Promise<{ logs: never[]; value?: unknown }>
  27. }
  28. const SCHEMA: StructuredOutputSchema = {
  29. type: 'object',
  30. properties: { answer: { type: 'number' }, note: { type: 'string' } },
  31. required: ['answer'],
  32. }
  33. /**
  34. * Real loop, scripted model, and inline fresh-conversation provider over the shared driver. Loading
  35. * spawn/fork here would create a dev-dependency cycle; their specs cover plugin integration while
  36. * this fixture isolates driver behavior and scripts the child's `structured_output` calls.
  37. */
  38. async function setup(script: Script, options: SetupOptions = {}) {
  39. const ctx = new Context()
  40. const adapter = new MockAdapter(script)
  41. await ctx.plugin(LlmService)
  42. await ctx.plugin(SessionStore)
  43. await ctx.plugin(SystemPrompt)
  44. await ctx.plugin(ToolRegistry, { mode: options.toolMode ?? 'native' })
  45. if (options.toolMode === 'code' || options.toolMode === 'both') {
  46. ctx.provide('codeRuntime', {
  47. language: 'typescript',
  48. isolation: 'test',
  49. run: options.codeRun ?? (() => Promise.resolve({ logs: [] })),
  50. } as never)
  51. }
  52. await ctx.plugin(AgentRegistry)
  53. await ctx.plugin(Invariants)
  54. await ctx.plugin(AgentLoop, { agents: [] })
  55. await ctx.plugin(SubagentService)
  56. const disposeProvider = ctx.subagents.registerProvider({
  57. name: 'spawn',
  58. capabilities: { outputSchema: true, depthLimit: true, toolFilter: false, persona: false },
  59. inheritsParentContext: false,
  60. start: (request: SubagentStartRequest) => startInProcessRun(request, {}),
  61. })
  62. ctx.llm.registerAdapter(['mock'], adapter)
  63. const parent = ctx.agentLoop.create(AgentId('parent'), { model: 'mock' })
  64. return { ctx, parent, adapter, disposeProvider }
  65. }
  66. function structuredRequest(parent: SubagentStartRequest['parent'], extra?: Partial<SubagentStartRequest>): SubagentStartRequest {
  67. return {
  68. prompt: [{ type: 'text', text: 'produce the answer' }],
  69. parent,
  70. signal: new AbortController().signal,
  71. outputSchema: SCHEMA,
  72. ...extra,
  73. }
  74. }
  75. /** The tool names of one recorded model request. */
  76. function toolNames(request: GenerateOptions): string[] {
  77. return (request.tools ?? []).map(tool => tool.name)
  78. }
  79. describe('in-process structured output', () => {
  80. it('captures a valid structured_output call and surfaces result.structured', async () => {
  81. const { ctx, parent } = await setup([
  82. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 42, note: 'done' }),
  83. ])
  84. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  85. const result = await run.result
  86. expect(result.stopReason).toBe('completed')
  87. expect(result.structured).toEqual({ answer: 42, note: 'done' })
  88. await run.dispose()
  89. })
  90. it('stops the turn after a successful capture — no extra model step is spent', async () => {
  91. const { ctx, parent, adapter } = await setup([
  92. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 1 }),
  93. textResponse('MUST NOT BE CONSUMED'),
  94. ])
  95. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  96. await run.result
  97. // Default continuation would run a second step after the tool call; the
  98. // structured runtime's turn-continuation veto stops the turn instead.
  99. expect(adapter.requests.length).toBe(1)
  100. await run.dispose()
  101. })
  102. it('denies tool calls that FOLLOW the capture in the same response — terminal means terminal', async () => {
  103. // One model response carrying structured_output FIRST and a side-effecting
  104. // call after it: the continuation veto only fires at step end, so without
  105. // the pre-execute deny the trailing call would still run after the final
  106. // answer was accepted.
  107. const response = [
  108. ...toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 5 }).slice(0, -2),
  109. { type: 'block-start', index: 1, blockType: 'tool-call' },
  110. { type: 'block-end', index: 1, block: { type: 'tool-call', id: CallId('c2'), name: 'side_effect', arguments: '{}' } },
  111. { type: 'usage', usage: { inputTokens: 10, outputTokens: 5 } },
  112. { type: 'finish', reason: { kind: 'tool-calls' } },
  113. ] as Script[number]
  114. const { ctx, parent } = await setup([response])
  115. let sideEffectRan = false
  116. ctx.tools.register({
  117. name: 'side_effect',
  118. description: 'probe',
  119. parameters: { type: 'object', properties: {} },
  120. execute(): Promise<ContentBlock[]> {
  121. sideEffectRan = true
  122. return Promise.resolve([{ type: 'text', text: 'ran' }])
  123. },
  124. })
  125. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  126. const result = await run.result
  127. expect(result.stopReason).toBe('completed')
  128. expect(result.structured).toEqual({ answer: 5 })
  129. // The deny skipped dispatch entirely: the probe body never ran.
  130. expect(sideEffectRan).toBe(false)
  131. await run.dispose()
  132. })
  133. it('a later prepended pre-execute listener cannot resurrect dispatch after capture', async () => {
  134. const response = [
  135. ...toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 5 }).slice(0, -2),
  136. { type: 'block-start', index: 1, blockType: 'tool-call' },
  137. { type: 'block-end', index: 1, block: { type: 'tool-call', id: CallId('c2'), name: 'side_effect', arguments: '{}' } },
  138. { type: 'usage', usage: { inputTokens: 10, outputTokens: 5 } },
  139. { type: 'finish', reason: { kind: 'tool-calls' } },
  140. ] as Script[number]
  141. const { ctx, parent } = await setup([response])
  142. let sideEffectRan = false
  143. ctx.tools.register({
  144. name: 'side_effect',
  145. description: 'probe',
  146. parameters: { type: 'object', properties: {} },
  147. execute(): Promise<ContentBlock[]> {
  148. sideEffectRan = true
  149. return Promise.resolve([{ type: 'text', text: 'ran' }])
  150. },
  151. })
  152. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  153. // Registered after the child and prepended: this listener returns allow
  154. // after every downstream pre-execute decision. The service-owned guard
  155. // runs after the waterfall and can only deny, so the body still cannot run.
  156. ctx.on('tools/pre-execute', async (_exec, next) => {
  157. await next()
  158. return { kind: 'allow' as const }
  159. }, { prepend: true })
  160. const result = await run.result
  161. expect(result.structured).toEqual({ answer: 5 })
  162. expect(sideEffectRan).toBe(false)
  163. const child = ctx.agents.get(run.id)
  164. const sideEffectResult = child?.session.events.find(event =>
  165. event.type === 'tool/result' && event.data.callId === 'c2')
  166. expect(sideEffectResult?.type === 'tool/result' && sideEffectResult.data.isError).toBe(true)
  167. await run.dispose()
  168. })
  169. it('leaves tool calls that PRECEDE the capture in the same response untouched', async () => {
  170. const response = [
  171. { type: 'block-start', index: 0, blockType: 'tool-call' },
  172. { type: 'block-end', index: 0, block: { type: 'tool-call', id: CallId('c1'), name: 'side_effect', arguments: '{}' } },
  173. ...toolCallResponse('c2', STRUCTURED_OUTPUT_TOOL, { answer: 6 }).map(chunk =>
  174. 'index' in chunk ? { ...chunk, index: 1 } : chunk),
  175. ] as Script[number]
  176. const { ctx, parent } = await setup([response])
  177. let sideEffectRan = false
  178. ctx.tools.register({
  179. name: 'side_effect',
  180. description: 'probe',
  181. parameters: { type: 'object', properties: {} },
  182. execute(): Promise<ContentBlock[]> {
  183. sideEffectRan = true
  184. return Promise.resolve([{ type: 'text', text: 'ran' }])
  185. },
  186. })
  187. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  188. const result = await run.result
  189. // The call ran BEFORE captured was set: the deny gate only guards the
  190. // window after the terminal answer landed.
  191. expect(sideEffectRan).toBe(true)
  192. expect(result.structured).toEqual({ answer: 6 })
  193. await run.dispose()
  194. })
  195. it('a later-prepended continuation wrapper cannot resurrect a captured turn', async () => {
  196. const { ctx, parent, adapter } = await setup([
  197. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 7 }),
  198. textResponse('MUST NOT BE CONSUMED'),
  199. ])
  200. ctx.on('agent/turn-continuation', () => Promise.resolve<ContinuationDecision>({ action: 'stop' }))
  201. let wrapperInstalled = false
  202. // Register before ready-only start: structured output is attached before session-start and the
  203. // loop. The wrapper waits for a downstream stop, rewrites it to continue, and must still lose
  204. // to the later terminal checkpoint.
  205. ctx.on('agent/session-start', (child) => {
  206. if (child === parent) return
  207. wrapperInstalled = true
  208. child.ctx.on('agent/turn-continuation', async (_subject, _turn, _decision, next): Promise<ContinuationDecision> => {
  209. const downstream = await next()
  210. expect(downstream).toEqual({ action: 'stop' })
  211. return { action: 'continue' }
  212. }, { prepend: true })
  213. })
  214. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  215. const result = await run.result
  216. expect(wrapperInstalled).toBe(true)
  217. expect(result.structured).toEqual({ answer: 7 })
  218. expect(result.stopReason).toBe('completed')
  219. expect(adapter.requests).toHaveLength(1)
  220. await run.dispose()
  221. })
  222. it('a continuation wrapper cannot carry steering past a captured terminal stop', async () => {
  223. const { ctx, parent, adapter } = await setup([
  224. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 9 }),
  225. textResponse('MUST NOT BE CONSUMED'),
  226. ])
  227. // A downstream policy stops, then a later wrapper delegates and queues steering that ordinary
  228. // folding would turn into continue. The terminal checkpoint must discard that steering.
  229. ctx.on('agent/turn-continuation', () => Promise.resolve<ContinuationDecision>({ action: 'stop' }))
  230. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  231. ctx.on('agent/session-start', (child) => {
  232. if (child.id !== run.id) return
  233. child.ctx.on('agent/turn-continuation', async (subject, _turn, _decision, next): Promise<ContinuationDecision> => {
  234. const downstream = await next()
  235. expect(downstream).toEqual({ action: 'stop' })
  236. subject.steer([{ type: 'text', text: 'late steering after downstream stop' }])
  237. return downstream
  238. }, { prepend: true })
  239. })
  240. const result = await run.result
  241. const child = ctx.agents.get(run.id)
  242. expect(result.structured).toEqual({ answer: 9 })
  243. expect(adapter.requests).toHaveLength(1)
  244. expect(child?.session.events.filter(event => event.type === 'turn/start')).toHaveLength(1)
  245. expect(child?.session.events.filter(event => event.type === 'steering/message')).toHaveLength(0)
  246. await run.dispose()
  247. })
  248. it('an invalid call gets an INVALID_ARGS isError result and the model retries in-turn', async () => {
  249. const { ctx, parent } = await setup([
  250. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 'not-a-number' }),
  251. toolCallResponse('c2', STRUCTURED_OUTPUT_TOOL, { answer: 7 }),
  252. ])
  253. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  254. const result = await run.result
  255. expect(result.structured).toEqual({ answer: 7 })
  256. expect(result.stopReason).toBe('completed')
  257. // The child's log carries the isError tool/result for the invalid call.
  258. const child = ctx.agents.get(run.id)!
  259. const results = child.session.events.filter(e => e.type === 'tool/result')
  260. expect(results.length).toBe(2)
  261. expect((results[0]!.data as { isError?: boolean }).isError).toBe(true)
  262. await run.dispose()
  263. })
  264. it('a clean finish without a capture is an immediate error to the parent — deliberately NO re-prompt', async () => {
  265. const { ctx, parent, adapter } = await setup([
  266. textResponse('here is my answer in prose'),
  267. textResponse('MUST NOT BE CONSUMED'),
  268. ])
  269. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  270. const result = await run.result
  271. expect(result.stopReason).toBe('error')
  272. expect(result.structured).toBeUndefined()
  273. // Exactly one model request and one user message: no nudge turn exists.
  274. expect(adapter.requests.length).toBe(1)
  275. const child = ctx.agents.get(run.id)!
  276. expect(child.session.events.filter(e => e.type === 'user/message').length).toBe(1)
  277. await run.dispose()
  278. })
  279. it('an errored child keeps its honest error result (no capture expected)', async () => {
  280. // Script exhaustion on the first call → the child turn errors.
  281. const { ctx, parent, adapter } = await setup([])
  282. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  283. const result = await run.result
  284. expect(result.stopReason).toBe('error')
  285. expect(adapter.requests.length).toBe(1)
  286. await run.dispose()
  287. })
  288. it('a cancel landing after a clean capture-less turn settles aborted, not error', async () => {
  289. const { ctx, parent } = await setup([textResponse('prose, no capture')])
  290. const controller = new AbortController()
  291. const run = await ctx.subagents.start('spawn', structuredRequest(parent, { signal: controller.signal }))
  292. // Cancel synchronously inside the turn's end recording: the cancel
  293. // contract outranks the schema shortfall, so the result maps to aborted.
  294. ctx.on('session/event', (session, event) => {
  295. const child = ctx.agents.get(run.id)
  296. if (session === child?.session && event.type === 'turn/end') controller.abort('cancelled at turn end')
  297. })
  298. const result = await run.result
  299. expect(result.stopReason).toBe('aborted')
  300. await run.dispose()
  301. })
  302. it('rejects a schema outside the subset loud, before any child exists', async () => {
  303. const { ctx, parent } = await setup([])
  304. await expect(ctx.subagents.start('spawn', structuredRequest(parent, {
  305. outputSchema: { type: 'object', oneOf: [] } as unknown as StructuredOutputSchema,
  306. }))).rejects.toThrow(/unsupported output schema/)
  307. expect(ctx.agents.get(AgentId('parent'))).toBeDefined()
  308. })
  309. it('a schema carrying non-JSON values fails as OutputSchemaError at the validation boundary', async () => {
  310. const { ctx, parent } = await setup([])
  311. // Semantic assertion runs before provider startup.
  312. await expect(ctx.subagents.start('spawn', structuredRequest(parent, {
  313. outputSchema: { type: 'object', default: () => {} } as unknown as StructuredOutputSchema,
  314. }))).rejects.toThrow(/unsupported output schema.*annotation must be JSON data/)
  315. })
  316. it('a post-execute BLOCK on the capture call denies the capture: log and result agree on failure', async () => {
  317. const { ctx, parent, adapter } = await setup([
  318. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 7 }),
  319. textResponse('continues after the blocked capture'),
  320. ])
  321. // A PostToolUse-style hook turns the tool body's provisional success into
  322. // the authoritative final error observed by the commit notification.
  323. ctx.on('tools/post-execute', (exec, _result, next) => {
  324. if (exec.name === STRUCTURED_OUTPUT_TOOL) {
  325. return Promise.resolve({ kind: 'block' as const, feedback: [{ type: 'text' as const, text: 'capture rejected by hook' }] })
  326. }
  327. return next()
  328. })
  329. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  330. const result = await run.result
  331. // No capture was committed: the run reports the schema shortfall...
  332. expect(result.structured).toBeUndefined()
  333. expect(result.stopReason).toBe('error')
  334. // ...the logged tool result is the blocked isError with the feedback...
  335. const child = ctx.agents.get(run.id)!
  336. const results = child.session.events.filter(e => e.type === 'tool/result')
  337. expect((results[0]!.data as { isError?: boolean }).isError).toBe(true)
  338. expect(JSON.stringify((results[0]!.data as { content: unknown }).content)).toContain('capture rejected by hook')
  339. // ...and the turn CONTINUED past the blocked call (no captured veto):
  340. // the model got to react to the failure with a second step.
  341. expect(adapter.requests.length).toBe(2)
  342. await run.dispose()
  343. })
  344. it('a post-execute accept-with-replacement still commits the capture', async () => {
  345. const { ctx, parent } = await setup([
  346. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 8 }),
  347. ])
  348. ctx.on('tools/post-execute', (exec, _result, next) => {
  349. if (exec.name === STRUCTURED_OUTPUT_TOOL) {
  350. return Promise.resolve({ kind: 'accept' as const, content: [{ type: 'text' as const, text: 'recorded (rewritten)' }] })
  351. }
  352. return next()
  353. })
  354. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  355. const result = await run.result
  356. expect(result.stopReason).toBe('completed')
  357. expect(result.structured).toEqual({ answer: 8 })
  358. await run.dispose()
  359. })
  360. it('commits only after a later prepended post-execute wrapper returns the authoritative result', async () => {
  361. const { ctx, parent } = await setup([
  362. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 8 }),
  363. textResponse('capture was rejected'),
  364. ])
  365. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  366. // Registered after attachment and prepended, so it wraps every listener
  367. // the child installed. It delegates first, then converts the apparent
  368. // capture success into the pipeline's authoritative failure.
  369. ctx.on('tools/post-execute', async (exec, _result, next) => {
  370. const downstream = await next()
  371. if (exec.name !== STRUCTURED_OUTPUT_TOOL) return downstream
  372. return { kind: 'block' as const, feedback: [{ type: 'text' as const, text: 'rejected after downstream' }] }
  373. }, { prepend: true })
  374. const result = await run.result
  375. expect(result.structured).toBeUndefined()
  376. expect(result.stopReason).toBe('error')
  377. const child = ctx.agents.get(run.id)
  378. const captureResult = child?.session.events.find(event =>
  379. event.type === 'tool/result' && event.data.callId === 'c1')
  380. expect(captureResult?.type === 'tool/result' && captureResult.data.isError).toBe(true)
  381. await run.dispose()
  382. })
  383. it('appends the structured instruction to the child REQUEST\'s system text (base prompt preserved)', async () => {
  384. const { ctx, parent, adapter } = await setup([toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 1 })])
  385. // A context-wide section stands in for the deployment persona: the
  386. // instruction must APPEND to the other scoped and global sections, not
  387. // replace them (AgentOptions has no prompt field — the instruction is an
  388. // ordinary child-scoped prompt registration).
  389. ctx.systemPrompt.section({ name: 'test:persona', order: 10, text: 'You are a counter.' })
  390. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  391. await run.result
  392. const childRequest = adapter.requests.at(-1)!
  393. expect(childRequest.system).toContain('You are a counter.')
  394. expect(childRequest.system!.endsWith(STRUCTURED_OUTPUT_INSTRUCTION)).toBe(true)
  395. expect(childRequest.system!.indexOf(STRUCTURED_OUTPUT_INSTRUCTION)).toBeGreaterThan(0)
  396. await run.dispose()
  397. })
  398. it('keeps pure Code Mode at one wire tool and exposes structured capture through the SDK only', async () => {
  399. const { ctx, parent, adapter } = await setup([
  400. toolCallResponse('c1', RUN_CODE_NAME, { code: 'return await tools.structured_output({ answer: 12 })' }),
  401. ], {
  402. toolMode: 'code',
  403. codeRun: async (request) => {
  404. const capture = request.bindings.at(0)?.functions[STRUCTURED_OUTPUT_TOOL]
  405. if (!capture) throw new Error('structured_output binding missing')
  406. await capture({ answer: 12 })
  407. return { logs: [], value: 'captured' }
  408. },
  409. })
  410. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  411. const result = await run.result
  412. expect(result.structured).toEqual({ answer: 12 })
  413. const request = adapter.requests[0]!
  414. expect(toolNames(request)).toEqual([RUN_CODE_NAME])
  415. expect(request.system).toContain('declare const tools:')
  416. expect(request.system).toContain('structured_output(args:')
  417. expect(request.system).toContain(STRUCTURED_OUTPUT_INSTRUCTION)
  418. await run.dispose()
  419. })
  420. it('discards a nested capture when the enclosing run_code execution fails', async () => {
  421. const { ctx, parent, adapter } = await setup([
  422. toolCallResponse('c1', RUN_CODE_NAME, { code: 'await tools.structured_output({ answer: 12 }); throw new Error("boom")' }),
  423. textResponse('outer code failed'),
  424. ], {
  425. toolMode: 'code',
  426. codeRun: async (request) => {
  427. const capture = request.bindings.at(0)?.functions[STRUCTURED_OUTPUT_TOOL]
  428. if (!capture) throw new Error('structured_output binding missing')
  429. await capture({ answer: 12 })
  430. return {
  431. logs: [],
  432. error: { kind: 'runtime', message: 'boom after capture' },
  433. } as never
  434. },
  435. })
  436. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  437. const result = await run.result
  438. expect(result.structured).toBeUndefined()
  439. expect(result.stopReason).toBe('error')
  440. expect(adapter.requests).toHaveLength(2)
  441. const child = ctx.agents.get(run.id)!
  442. const outer = child.session.events.find(event =>
  443. event.type === 'tool/result' && event.data.callId === CallId('c1'))
  444. expect(outer?.type === 'tool/result' && outer.data.isError).toBe(true)
  445. await run.dispose()
  446. })
  447. it('discards a nested capture when post-policy blocks the enclosing run_code result', async () => {
  448. const { ctx, parent, adapter } = await setup([
  449. toolCallResponse('c1', RUN_CODE_NAME, { code: 'return await tools.structured_output({ answer: 12 })' }),
  450. textResponse('outer code was blocked'),
  451. ], {
  452. toolMode: 'code',
  453. codeRun: async (request) => {
  454. const capture = request.bindings.at(0)?.functions[STRUCTURED_OUTPUT_TOOL]
  455. if (!capture) throw new Error('structured_output binding missing')
  456. await capture({ answer: 12 })
  457. return { logs: [], value: 'captured' }
  458. },
  459. })
  460. ctx.on('tools/post-execute', (exec, _result, next) => exec.name === RUN_CODE_NAME
  461. ? Promise.resolve({ kind: 'block' as const, feedback: [{ type: 'text' as const, text: 'outer blocked' }] })
  462. : next())
  463. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  464. const result = await run.result
  465. expect(result.structured).toBeUndefined()
  466. expect(result.stopReason).toBe('error')
  467. expect(adapter.requests).toHaveLength(2)
  468. await run.dispose()
  469. })
  470. it('the instruction rides ONLY structured requests: appended for the child, absent for a plain agent', async () => {
  471. const { ctx, parent, adapter } = await setup([
  472. textResponse('parent answer'),
  473. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 1 }),
  474. ])
  475. parent.send([{ type: 'text', text: 'hello' }])
  476. await parent.whenIdle()
  477. expect(adapter.requests[0]!.system ?? '').not.toContain(STRUCTURED_OUTPUT_INSTRUCTION)
  478. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  479. await run.result
  480. // The loop always assembles a base prompt (the harness identity section),
  481. // so the instruction APPENDS — never replaces.
  482. const childSystem = adapter.requests.at(-1)!.system!
  483. expect(childSystem.endsWith(STRUCTURED_OUTPUT_INSTRUCTION)).toBe(true)
  484. expect(childSystem.length).toBeGreaterThan(STRUCTURED_OUTPUT_INSTRUCTION.length)
  485. await run.dispose()
  486. })
  487. describe('scoped registration (each child owns its capture tool)', () => {
  488. it('a plain agent never sees the tool: nothing is registered globally at all', async () => {
  489. const { ctx, parent, adapter } = await setup([textResponse('parent answer')])
  490. parent.send([{ type: 'text', text: 'hello' }])
  491. await parent.whenIdle()
  492. // Scoped registration: the global view has no capture tool, ever.
  493. expect(ctx.tools.get(STRUCTURED_OUTPUT_TOOL)).toBeUndefined()
  494. expect(toolNames(adapter.requests[0]!)).not.toContain(STRUCTURED_OUTPUT_TOOL)
  495. })
  496. it('a structured child sees structured_output with ITS schema; a plain agent never sees the tool', async () => {
  497. const { ctx, parent, adapter } = await setup([
  498. // Parent turn (a plain agent): must NOT see the tool.
  499. textResponse('parent answer'),
  500. // Child turn: must see it, with the run's schema.
  501. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 42 }),
  502. ])
  503. parent.send([{ type: 'text', text: 'hello' }])
  504. await parent.whenIdle()
  505. expect(toolNames(adapter.requests[0]!)).not.toContain(STRUCTURED_OUTPUT_TOOL)
  506. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  507. await run.result
  508. const childRequest = adapter.requests[1]!
  509. expect(toolNames(childRequest)).toContain(STRUCTURED_OUTPUT_TOOL)
  510. const entry = childRequest.tools!.find(tool => tool.name === STRUCTURED_OUTPUT_TOOL)!
  511. expect(entry.parameters).toEqual(SCHEMA)
  512. await run.dispose()
  513. })
  514. it('two concurrent structured children each see their OWN schema', async () => {
  515. const otherSchema: StructuredOutputSchema = {
  516. type: 'object',
  517. properties: { verdict: { type: 'string', enum: ['real', 'bogus'] } },
  518. required: ['verdict'],
  519. }
  520. const { ctx, parent, adapter } = await setup([
  521. (options: GenerateOptions) => {
  522. // Answer with whatever schema this child was given — proves each
  523. // request carried the right one regardless of scheduling order.
  524. const entry = options.tools!.find(tool => tool.name === STRUCTURED_OUTPUT_TOOL)!
  525. const args = 'verdict' in (entry.parameters.properties as Record<string, unknown>)
  526. ? { verdict: 'real' }
  527. : { answer: 1 }
  528. return toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, args)
  529. },
  530. (options: GenerateOptions) => {
  531. const entry = options.tools!.find(tool => tool.name === STRUCTURED_OUTPUT_TOOL)!
  532. const args = 'verdict' in (entry.parameters.properties as Record<string, unknown>)
  533. ? { verdict: 'real' }
  534. : { answer: 1 }
  535. return toolCallResponse('c2', STRUCTURED_OUTPUT_TOOL, args)
  536. },
  537. ])
  538. const runA = await ctx.subagents.start('spawn', structuredRequest(parent))
  539. const runB = await ctx.subagents.start('spawn', structuredRequest(parent, { outputSchema: otherSchema }))
  540. const [a, b] = await Promise.all([runA.result, runB.result])
  541. expect(a.structured).toEqual({ answer: 1 })
  542. expect(b.structured).toEqual({ verdict: 'real' })
  543. const schemas = adapter.requests.map(request =>
  544. request.tools!.find(tool => tool.name === STRUCTURED_OUTPUT_TOOL)!.parameters)
  545. expect(schemas).toContainEqual(SCHEMA)
  546. expect(schemas).toContainEqual(otherSchema)
  547. await runA.dispose()
  548. await runB.dispose()
  549. })
  550. it('places the capture tool and instruction in their canonical orders', async () => {
  551. const { ctx, parent, adapter } = await setup([
  552. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 7 }),
  553. ])
  554. // A global tool sorts lexicographically after structured_output, while a
  555. // global section above the 190 band follows the capture instruction.
  556. ctx.tools.register({
  557. name: 'zz_probe',
  558. description: 'probe',
  559. parameters: { type: 'object', properties: {} },
  560. execute: () => Promise.resolve([{ type: 'text', text: 'x' }]),
  561. })
  562. ctx.systemPrompt.section({ name: 'after-band', order: 200, text: 'AFTER-BAND' })
  563. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  564. await run.result
  565. const request = adapter.requests[0]!
  566. const names = toolNames(request)
  567. expect(names.indexOf(STRUCTURED_OUTPUT_TOOL)).toBeGreaterThanOrEqual(0)
  568. expect(names.indexOf(STRUCTURED_OUTPUT_TOOL)).toBeLessThan(names.indexOf('zz_probe'))
  569. const system = request.system ?? ''
  570. const instructionAt = system.indexOf(STRUCTURED_OUTPUT_INSTRUCTION)
  571. expect(instructionAt).toBeGreaterThanOrEqual(0)
  572. expect(system.indexOf('AFTER-BAND')).toBeGreaterThan(instructionAt)
  573. await run.dispose()
  574. })
  575. it('a non-structured agent request keeps tools ABSENT when it had none (no tools: [] materialized)', async () => {
  576. const { parent, adapter } = await setup([textResponse('plain')])
  577. parent.send([{ type: 'text', text: 'q' }])
  578. await parent.whenIdle()
  579. const request = adapter.requests[0]!
  580. expect(request.tools).toBeUndefined()
  581. await new Promise(resolve => setTimeout(resolve, 0))
  582. })
  583. it('registrations ride the child fiber: disposing the run removes them; a provider reload mid-run cannot', async () => {
  584. const { ctx, parent, disposeProvider } = await setup([
  585. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 4 }),
  586. ])
  587. expect(ctx.tools.get(STRUCTURED_OUTPUT_TOOL)).toBeUndefined()
  588. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  589. // A backend hot-reload mid-run must not unregister the capture tool out
  590. // from under the live child: the registration rides the CHILD's fiber.
  591. disposeProvider()
  592. const result = await run.result
  593. expect(result.structured).toEqual({ answer: 4 })
  594. const child = ctx.agents.get(run.id)!
  595. expect(ctx.tools.get(STRUCTURED_OUTPUT_TOOL, child)).toBeDefined()
  596. await run.dispose()
  597. // Child disposed ⇒ its scoped registrations are gone.
  598. expect(ctx.tools.get(STRUCTURED_OUTPUT_TOOL, child)).toBeUndefined()
  599. })
  600. })
  601. it('a structured_output call from an agent WITHOUT a structured run is UNKNOWN_TOOL (the tool does not exist for it)', async () => {
  602. const { ctx, parent } = await setup([])
  603. const result = await ctx.tools.execute({
  604. callId: 'x' as never,
  605. name: STRUCTURED_OUTPUT_TOOL,
  606. arguments: { answer: 1 },
  607. agent: parent,
  608. })
  609. expect(result.isError).toBe(true)
  610. expect(result.error?.code).toBe('UNKNOWN_TOOL')
  611. })
  612. it('a structured_output call with NO calling agent at all is UNKNOWN_TOOL', async () => {
  613. const { ctx } = await setup([])
  614. const result = await ctx.tools.execute({
  615. callId: 'x' as never,
  616. name: STRUCTURED_OUTPUT_TOOL,
  617. arguments: { answer: 1 },
  618. })
  619. expect(result.isError).toBe(true)
  620. expect(result.error?.code).toBe('UNKNOWN_TOOL')
  621. })
  622. it('a failed execution stage is discarded and never promoted by a later call', async () => {
  623. const { ctx, parent } = await setup([
  624. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 1 }),
  625. ])
  626. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  627. // A prepended post-execute listener blocks the first capture without
  628. // delegating. The final-result notification discards that execution's
  629. // stage when it observes the error.
  630. let blocks = 1
  631. ctx.on('tools/post-execute', (exec, _result, next) => {
  632. if (exec.name === STRUCTURED_OUTPUT_TOOL && blocks > 0) {
  633. blocks -= 1
  634. return Promise.resolve({ kind: 'block' as const, feedback: [{ type: 'text' as const, text: 'rejected' }] })
  635. }
  636. return next()
  637. }, { prepend: true })
  638. const result = await run.result
  639. const child = ctx.agents.get(run.id)!
  640. // The blocked capture must NOT surface as structured success…
  641. expect(result.stopReason).toBe('error')
  642. expect(result.structured).toBeUndefined()
  643. // …and a LATER invalid call (its own body staged nothing) must not
  644. // resurrect c1's discarded value: drive the pipeline directly.
  645. const invalid = await ctx.tools.execute({
  646. callId: 'c2' as never,
  647. name: STRUCTURED_OUTPUT_TOOL,
  648. arguments: { answer: 'not-a-number' },
  649. agent: child,
  650. })
  651. expect(invalid.isError).toBe(true)
  652. // A fresh valid call still captures ITS OWN value.
  653. const valid = await ctx.tools.execute({
  654. callId: 'c3' as never,
  655. name: STRUCTURED_OUTPUT_TOOL,
  656. arguments: { answer: 9 },
  657. agent: child,
  658. })
  659. expect(valid.isError).toBeFalsy()
  660. await run.dispose()
  661. })
  662. it('reusing a failed execution\'s call id never promotes its discarded stage', async () => {
  663. const { ctx, parent } = await setup([
  664. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 1 }),
  665. ])
  666. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  667. // Block the first capture after its body stages a value. Its final error
  668. // discards that execution's stage.
  669. let blocks = 1
  670. ctx.on('tools/post-execute', (exec, _result, next) => {
  671. if (exec.name === STRUCTURED_OUTPUT_TOOL && blocks > 0) {
  672. blocks -= 1
  673. return Promise.resolve({ kind: 'block' as const, feedback: [{ type: 'text' as const, text: 'rejected' }] })
  674. }
  675. return next()
  676. }, { prepend: true })
  677. await run.result
  678. const child = ctx.agents.get(run.id)!
  679. // A SECOND capture call with the SAME call id whose body never stages
  680. // (invalid args throw before the stage): the discarded value must not ride
  681. // its acceptance.
  682. const reused = await ctx.tools.execute({
  683. callId: 'c1' as never,
  684. name: STRUCTURED_OUTPUT_TOOL,
  685. arguments: { answer: 'not-a-number' },
  686. agent: child,
  687. })
  688. expect(reused.isError).toBe(true)
  689. // Nothing was ever committed: a fresh valid call is still required.
  690. const valid = await ctx.tools.execute({
  691. callId: 'c1' as never,
  692. name: STRUCTURED_OUTPUT_TOOL,
  693. arguments: { answer: 5 },
  694. agent: child,
  695. })
  696. expect(valid.isError).toBeFalsy()
  697. await run.dispose()
  698. })
  699. it('a pre-execute deny with call-id reuse cannot promote another execution\'s stage', async () => {
  700. const { ctx, parent } = await setup([
  701. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 1 }),
  702. ])
  703. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  704. // Discard the first capture's stage via a final post-execute block.
  705. let blocks = 1
  706. ctx.on('tools/post-execute', (exec, _result, next) => {
  707. if (exec.name === STRUCTURED_OUTPUT_TOOL && blocks > 0) {
  708. blocks -= 1
  709. return Promise.resolve({ kind: 'block' as const, feedback: [{ type: 'text' as const, text: 'rejected' }] })
  710. }
  711. return next()
  712. }, { prepend: true })
  713. await run.result
  714. const child = ctx.agents.get(run.id)!
  715. // A prepended pre-execute deny skips the body, while the denied call still
  716. // reaches the final notification with the same adapter-minted call id.
  717. const offDeny = ctx.on('tools/pre-execute', (exec) => {
  718. if (exec.name === STRUCTURED_OUTPUT_TOOL) {
  719. return Promise.resolve({ kind: 'deny' as const, reason: 'outer veto' })
  720. }
  721. return undefined as never
  722. }, { prepend: true })
  723. const denied = await ctx.tools.execute({
  724. callId: 'c1' as never,
  725. name: STRUCTURED_OUTPUT_TOOL,
  726. arguments: { answer: 2 },
  727. agent: child,
  728. })
  729. expect(denied.isError).toBe(true)
  730. offDeny()
  731. // The discarded value was never promoted: a fresh valid call is required
  732. // (and succeeds, proving the runtime is not wedged).
  733. const valid = await ctx.tools.execute({
  734. callId: 'c1' as never,
  735. name: STRUCTURED_OUTPUT_TOOL,
  736. arguments: { answer: 5 },
  737. agent: child,
  738. })
  739. expect(valid.isError).toBeFalsy()
  740. await run.dispose()
  741. })
  742. })