structured.spec.ts 36 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785
  1. import { describe, expect, it } from 'vitest'
  2. import { Context } from 'cordis'
  3. import { CallId, type ContentBlock, type GenerateOptions } from '@deepseek-ai/dsh-llm'
  4. import type { ContinuationDecision } from '@deepseek-ai/dsh-agent'
  5. import { SessionId } from '@deepseek-ai/dsh-session'
  6. import AgentLoop from '@deepseek-ai/dsh-agent-loop'
  7. import { mountAgentLoopTestDependencies } from '@deepseek-ai/dsh-agent-loop-testkit'
  8. import * as Invariants from '@deepseek-ai/dsh-invariants'
  9. import SubagentService, { type SubagentStartRequest } from '@deepseek-ai/dsh-subagent'
  10. import type { Config as ToolConfig, StructuredOutputSchema } from '@deepseek-ai/dsh-tools'
  11. import { RUN_CODE_NAME } from '@deepseek-ai/dsh-tools'
  12. import { MockAdapter, textResponse, toolCallResponse } from '../../../core/agent-loop/tests/mock-adapter.ts'
  13. import { startInProcessRun } from '../src/index.ts'
  14. import {
  15. STRUCTURED_OUTPUT_INSTRUCTION,
  16. STRUCTURED_OUTPUT_TOOL,
  17. } from '../src/structured.ts'
  18. type Script = ConstructorParameters<typeof MockAdapter>[0]
  19. interface CodeRunRequestLike {
  20. bindings: { global: string; functions: Record<string, (args: unknown) => Promise<unknown>> }[]
  21. }
  22. interface SetupOptions {
  23. toolMode?: ToolConfig['mode']
  24. codeRun?: (request: CodeRunRequestLike) => Promise<{ logs: never[]; value?: unknown }>
  25. }
  26. const SCHEMA: StructuredOutputSchema = {
  27. type: 'object',
  28. properties: { answer: { type: 'number' }, note: { type: 'string' } },
  29. required: ['answer'],
  30. }
  31. /**
  32. * Real loop, scripted model, and inline fresh-conversation provider over the shared driver. Loading
  33. * spawn/fork here would create a dev-dependency cycle; their specs cover plugin integration while
  34. * this fixture isolates driver behavior and scripts the child's `structured_output` calls.
  35. */
  36. async function setup(script: Script, options: SetupOptions = {}) {
  37. const ctx = new Context()
  38. const adapter = new MockAdapter(script)
  39. await mountAgentLoopTestDependencies(ctx, {
  40. tools: { mode: options.toolMode ?? 'native' },
  41. })
  42. if (options.toolMode === 'code' || options.toolMode === 'both') {
  43. ctx.provide('codeRuntime', {
  44. language: 'typescript',
  45. isolation: 'test',
  46. run: options.codeRun ?? (() => Promise.resolve({ logs: [] })),
  47. } as never)
  48. }
  49. await ctx.plugin(Invariants)
  50. await ctx.plugin(AgentLoop, { agents: [] })
  51. await ctx.plugin(SubagentService)
  52. const disposeProvider = ctx.subagents.registerProvider({
  53. name: 'spawn',
  54. capabilities: { outputSchema: true, depthLimit: true, toolFilter: false, persona: false },
  55. inheritsParentContext: false,
  56. start: (request: SubagentStartRequest) => startInProcessRun(request, {}),
  57. })
  58. ctx.llm.registerAdapter(['mock'], adapter)
  59. const parent = ctx.agentLoop.create(SessionId('parent'), { provider: 'mock', model: 'mock' })
  60. return { ctx, parent, adapter, disposeProvider }
  61. }
  62. function structuredRequest(parent: SubagentStartRequest['parent'], extra?: Partial<SubagentStartRequest>): SubagentStartRequest {
  63. return {
  64. prompt: [{ type: 'text', text: 'produce the answer' }],
  65. parent,
  66. signal: new AbortController().signal,
  67. outputSchema: SCHEMA,
  68. ...extra,
  69. }
  70. }
  71. /** The tool names of one recorded model request. */
  72. function toolNames(request: GenerateOptions): string[] {
  73. return (request.tools ?? []).map(tool => tool.name)
  74. }
  75. describe('in-process structured output', () => {
  76. it('captures a valid structured_output call and surfaces result.structured', async () => {
  77. const { ctx, parent } = await setup([
  78. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 42, note: 'done' }),
  79. ])
  80. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  81. const result = await run.result
  82. expect(result.stopReason).toBe('completed')
  83. expect(result.structured).toEqual({ answer: 42, note: 'done' })
  84. await run.dispose()
  85. })
  86. it('stops the turn after a successful capture — no extra model step is spent', async () => {
  87. const { ctx, parent, adapter } = await setup([
  88. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 1 }),
  89. textResponse('MUST NOT BE CONSUMED'),
  90. ])
  91. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  92. await run.result
  93. // Default continuation would run a second step after the tool call; the
  94. // structured runtime's turn-continuation veto stops the turn instead.
  95. expect(adapter.requests.length).toBe(1)
  96. await run.dispose()
  97. })
  98. it('denies tool calls that FOLLOW the capture in the same response — terminal means terminal', async () => {
  99. // One model response carrying structured_output FIRST and a side-effecting
  100. // call after it: the continuation veto only fires at step end, so without
  101. // the pre-execute deny the trailing call would still run after the final
  102. // answer was accepted.
  103. const response = [
  104. ...toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 5 }).slice(0, -2),
  105. { type: 'block-start', index: 1, blockType: 'tool-call' },
  106. { type: 'block-end', index: 1, block: { type: 'tool-call', id: CallId('c2'), name: 'side_effect', arguments: '{}' } },
  107. { type: 'usage', usage: { inputTokens: 10, outputTokens: 5 } },
  108. { type: 'finish', reason: { kind: 'tool-calls' } },
  109. ] as Script[number]
  110. const { ctx, parent } = await setup([response])
  111. let sideEffectRan = false
  112. ctx.tools.register({
  113. name: 'side_effect',
  114. description: 'probe',
  115. parameters: { type: 'object', properties: {} },
  116. execute(): Promise<ContentBlock[]> {
  117. sideEffectRan = true
  118. return Promise.resolve([{ type: 'text', text: 'ran' }])
  119. },
  120. })
  121. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  122. const result = await run.result
  123. expect(result.stopReason).toBe('completed')
  124. expect(result.structured).toEqual({ answer: 5 })
  125. // The deny skipped dispatch entirely: the probe body never ran.
  126. expect(sideEffectRan).toBe(false)
  127. await run.dispose()
  128. })
  129. it('a later prepended pre-execute listener cannot resurrect dispatch after capture', async () => {
  130. const response = [
  131. ...toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 5 }).slice(0, -2),
  132. { type: 'block-start', index: 1, blockType: 'tool-call' },
  133. { type: 'block-end', index: 1, block: { type: 'tool-call', id: CallId('c2'), name: 'side_effect', arguments: '{}' } },
  134. { type: 'usage', usage: { inputTokens: 10, outputTokens: 5 } },
  135. { type: 'finish', reason: { kind: 'tool-calls' } },
  136. ] as Script[number]
  137. const { ctx, parent } = await setup([response])
  138. let sideEffectRan = false
  139. ctx.tools.register({
  140. name: 'side_effect',
  141. description: 'probe',
  142. parameters: { type: 'object', properties: {} },
  143. execute(): Promise<ContentBlock[]> {
  144. sideEffectRan = true
  145. return Promise.resolve([{ type: 'text', text: 'ran' }])
  146. },
  147. })
  148. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  149. // Registered after the child and prepended: this listener returns allow
  150. // after every downstream pre-execute decision. The service-owned guard
  151. // runs after the waterfall and can only deny, so the body still cannot run.
  152. ctx.on('tools/pre-execute', async (_exec, next) => {
  153. await next()
  154. return { kind: 'allow' as const }
  155. }, { prepend: true })
  156. const result = await run.result
  157. expect(result.structured).toEqual({ answer: 5 })
  158. expect(sideEffectRan).toBe(false)
  159. const child = ctx.agents.get(run.id)
  160. const sideEffectResult = child?.session.events.find(event =>
  161. event.type === 'tool/result' && event.data.callId === 'c2')
  162. expect(sideEffectResult?.type === 'tool/result' && sideEffectResult.data.isError).toBe(true)
  163. await run.dispose()
  164. })
  165. it('leaves tool calls that PRECEDE the capture in the same response untouched', async () => {
  166. const response = [
  167. { type: 'block-start', index: 0, blockType: 'tool-call' },
  168. { type: 'block-end', index: 0, block: { type: 'tool-call', id: CallId('c1'), name: 'side_effect', arguments: '{}' } },
  169. ...toolCallResponse('c2', STRUCTURED_OUTPUT_TOOL, { answer: 6 }).map(chunk =>
  170. 'index' in chunk ? { ...chunk, index: 1 } : chunk),
  171. ] as Script[number]
  172. const { ctx, parent } = await setup([response])
  173. let sideEffectRan = false
  174. ctx.tools.register({
  175. name: 'side_effect',
  176. description: 'probe',
  177. parameters: { type: 'object', properties: {} },
  178. execute(): Promise<ContentBlock[]> {
  179. sideEffectRan = true
  180. return Promise.resolve([{ type: 'text', text: 'ran' }])
  181. },
  182. })
  183. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  184. const result = await run.result
  185. // The call ran BEFORE captured was set: the deny gate only guards the
  186. // window after the terminal answer landed.
  187. expect(sideEffectRan).toBe(true)
  188. expect(result.structured).toEqual({ answer: 6 })
  189. await run.dispose()
  190. })
  191. it('a later-prepended continuation wrapper cannot resurrect a captured turn', async () => {
  192. const { ctx, parent, adapter } = await setup([
  193. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 7 }),
  194. textResponse('MUST NOT BE CONSUMED'),
  195. ])
  196. ctx.on('agent/turn-continuation', () => Promise.resolve<ContinuationDecision>({ action: 'stop' }))
  197. let wrapperInstalled = false
  198. // Register before ready-only start: structured output is attached before session-start and the
  199. // loop. The wrapper waits for a downstream stop, rewrites it to continue, and must still lose
  200. // to the later terminal checkpoint.
  201. ctx.on('agent/session-start', (child) => {
  202. if (child === parent) return
  203. wrapperInstalled = true
  204. child.ctx.on('agent/turn-continuation', async (_subject, _turn, _decision, next): Promise<ContinuationDecision> => {
  205. const downstream = await next()
  206. expect(downstream).toEqual({ action: 'stop' })
  207. return { action: 'continue' }
  208. }, { prepend: true })
  209. })
  210. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  211. const result = await run.result
  212. expect(wrapperInstalled).toBe(true)
  213. expect(result.structured).toEqual({ answer: 7 })
  214. expect(result.stopReason).toBe('completed')
  215. expect(adapter.requests).toHaveLength(1)
  216. await run.dispose()
  217. })
  218. it('a continuation wrapper cannot carry steering past a captured terminal stop', async () => {
  219. const { ctx, parent, adapter } = await setup([
  220. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 9 }),
  221. textResponse('MUST NOT BE CONSUMED'),
  222. ])
  223. // A downstream policy stops, then a later wrapper delegates and queues steering that ordinary
  224. // folding would turn into continue. The terminal checkpoint must discard that steering.
  225. ctx.on('agent/turn-continuation', () => Promise.resolve<ContinuationDecision>({ action: 'stop' }))
  226. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  227. ctx.on('agent/session-start', (child) => {
  228. if (child.id !== run.id) return
  229. child.ctx.on('agent/turn-continuation', async (subject, _turn, _decision, next): Promise<ContinuationDecision> => {
  230. const downstream = await next()
  231. expect(downstream).toEqual({ action: 'stop' })
  232. subject.steer([{ type: 'text', text: 'late steering after downstream stop' }])
  233. return downstream
  234. }, { prepend: true })
  235. })
  236. const result = await run.result
  237. const child = ctx.agents.get(run.id)
  238. expect(result.structured).toEqual({ answer: 9 })
  239. expect(adapter.requests).toHaveLength(1)
  240. expect(child?.session.events.filter(event => event.type === 'turn/start')).toHaveLength(1)
  241. expect(child?.session.events.filter(event => event.type === 'steering/message')).toHaveLength(0)
  242. await run.dispose()
  243. })
  244. it('an invalid call gets an INVALID_ARGS isError result and the model retries in-turn', async () => {
  245. const { ctx, parent } = await setup([
  246. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 'not-a-number' }),
  247. toolCallResponse('c2', STRUCTURED_OUTPUT_TOOL, { answer: 7 }),
  248. ])
  249. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  250. const result = await run.result
  251. expect(result.structured).toEqual({ answer: 7 })
  252. expect(result.stopReason).toBe('completed')
  253. // The child's log carries the isError tool/result for the invalid call.
  254. const child = ctx.agents.get(run.id)!
  255. const results = child.session.events.filter(e => e.type === 'tool/result')
  256. expect(results.length).toBe(2)
  257. expect((results[0]!.data as { isError?: boolean }).isError).toBe(true)
  258. await run.dispose()
  259. })
  260. it('a clean finish without a capture is an immediate error to the parent — deliberately NO re-prompt', async () => {
  261. const { ctx, parent, adapter } = await setup([
  262. textResponse('here is my answer in prose'),
  263. textResponse('MUST NOT BE CONSUMED'),
  264. ])
  265. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  266. const result = await run.result
  267. expect(result.stopReason).toBe('error')
  268. expect(result.structured).toBeUndefined()
  269. // Exactly one model request and one user message: no nudge turn exists.
  270. expect(adapter.requests.length).toBe(1)
  271. const child = ctx.agents.get(run.id)!
  272. expect(child.session.events.filter(e => e.type === 'user/message').length).toBe(1)
  273. await run.dispose()
  274. })
  275. it('an errored child keeps its honest error result (no capture expected)', async () => {
  276. // Script exhaustion on the first call → the child turn errors.
  277. const { ctx, parent, adapter } = await setup([])
  278. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  279. const result = await run.result
  280. expect(result.stopReason).toBe('error')
  281. expect(adapter.requests.length).toBe(1)
  282. await run.dispose()
  283. })
  284. it('a cancel landing after a clean capture-less turn settles aborted, not error', async () => {
  285. const { ctx, parent } = await setup([textResponse('prose, no capture')])
  286. const controller = new AbortController()
  287. const run = await ctx.subagents.start('spawn', structuredRequest(parent, { signal: controller.signal }))
  288. // Cancel synchronously inside the turn's end recording: the cancel
  289. // contract outranks the schema shortfall, so the result maps to aborted.
  290. ctx.on('session/event', (session, event) => {
  291. const child = ctx.agents.get(run.id)
  292. if (session === child?.session && event.type === 'turn/end') controller.abort('cancelled at turn end')
  293. })
  294. const result = await run.result
  295. expect(result.stopReason).toBe('aborted')
  296. await run.dispose()
  297. })
  298. it('rejects a schema outside the subset loud, before any child exists', async () => {
  299. const { ctx, parent } = await setup([])
  300. await expect(ctx.subagents.start('spawn', structuredRequest(parent, {
  301. outputSchema: { type: 'object', oneOf: [] } as unknown as StructuredOutputSchema,
  302. }))).rejects.toThrow(/unsupported output schema/)
  303. expect(ctx.agents.get(SessionId('parent'))).toBeDefined()
  304. })
  305. it('a schema carrying non-JSON values fails as OutputSchemaError at the validation boundary', async () => {
  306. const { ctx, parent } = await setup([])
  307. // Semantic assertion runs before provider startup.
  308. await expect(ctx.subagents.start('spawn', structuredRequest(parent, {
  309. outputSchema: { type: 'object', default: () => {} } as unknown as StructuredOutputSchema,
  310. }))).rejects.toThrow(/unsupported output schema.*annotation must be JSON data/)
  311. })
  312. it('a post-execute BLOCK on the capture call denies the capture: log and result agree on failure', async () => {
  313. const { ctx, parent, adapter } = await setup([
  314. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 7 }),
  315. textResponse('continues after the blocked capture'),
  316. ])
  317. // A PostToolUse-style hook turns the tool body's provisional success into
  318. // the authoritative final error observed by the commit notification.
  319. ctx.on('tools/post-execute', (exec, _result, next) => {
  320. if (exec.name === STRUCTURED_OUTPUT_TOOL) {
  321. return Promise.resolve({ kind: 'block' as const, feedback: [{ type: 'text' as const, text: 'capture rejected by hook' }] })
  322. }
  323. return next()
  324. })
  325. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  326. const result = await run.result
  327. // No capture was committed: the run reports the schema shortfall...
  328. expect(result.structured).toBeUndefined()
  329. expect(result.stopReason).toBe('error')
  330. // ...the logged tool result is the blocked isError with the feedback...
  331. const child = ctx.agents.get(run.id)!
  332. const results = child.session.events.filter(e => e.type === 'tool/result')
  333. expect((results[0]!.data as { isError?: boolean }).isError).toBe(true)
  334. expect(JSON.stringify((results[0]!.data as { content: unknown }).content)).toContain('capture rejected by hook')
  335. // ...and the turn CONTINUED past the blocked call (no captured veto):
  336. // the model got to react to the failure with a second step.
  337. expect(adapter.requests.length).toBe(2)
  338. await run.dispose()
  339. })
  340. it('a post-execute accept-with-replacement still commits the capture', async () => {
  341. const { ctx, parent } = await setup([
  342. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 8 }),
  343. ])
  344. ctx.on('tools/post-execute', (exec, _result, next) => {
  345. if (exec.name === STRUCTURED_OUTPUT_TOOL) {
  346. return Promise.resolve({ kind: 'accept' as const, content: [{ type: 'text' as const, text: 'recorded (rewritten)' }] })
  347. }
  348. return next()
  349. })
  350. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  351. const result = await run.result
  352. expect(result.stopReason).toBe('completed')
  353. expect(result.structured).toEqual({ answer: 8 })
  354. await run.dispose()
  355. })
  356. it('commits only after a later prepended post-execute wrapper returns the authoritative result', async () => {
  357. const { ctx, parent } = await setup([
  358. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 8 }),
  359. textResponse('capture was rejected'),
  360. ])
  361. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  362. // Registered after attachment and prepended, so it wraps every listener
  363. // the child installed. It delegates first, then converts the apparent
  364. // capture success into the pipeline's authoritative failure.
  365. ctx.on('tools/post-execute', async (exec, _result, next) => {
  366. const downstream = await next()
  367. if (exec.name !== STRUCTURED_OUTPUT_TOOL) return downstream
  368. return { kind: 'block' as const, feedback: [{ type: 'text' as const, text: 'rejected after downstream' }] }
  369. }, { prepend: true })
  370. const result = await run.result
  371. expect(result.structured).toBeUndefined()
  372. expect(result.stopReason).toBe('error')
  373. const child = ctx.agents.get(run.id)
  374. const captureResult = child?.session.events.find(event =>
  375. event.type === 'tool/result' && event.data.callId === 'c1')
  376. expect(captureResult?.type === 'tool/result' && captureResult.data.isError).toBe(true)
  377. await run.dispose()
  378. })
  379. it('appends the structured instruction to the child REQUEST\'s system text (base prompt preserved)', async () => {
  380. const { ctx, parent, adapter } = await setup([toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 1 })])
  381. // A context-wide section stands in for the deployment persona: the
  382. // instruction must APPEND to the other scoped and global sections, not
  383. // replace them (AgentOptions has no prompt field — the instruction is an
  384. // ordinary child-scoped prompt registration).
  385. ctx.systemPrompt.section({ name: 'test:persona', order: 10, text: 'You are a counter.' })
  386. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  387. await run.result
  388. const childRequest = adapter.requests.at(-1)!
  389. expect(childRequest.system).toContain('You are a counter.')
  390. expect(childRequest.system!.endsWith(STRUCTURED_OUTPUT_INSTRUCTION)).toBe(true)
  391. expect(childRequest.system!.indexOf(STRUCTURED_OUTPUT_INSTRUCTION)).toBeGreaterThan(0)
  392. await run.dispose()
  393. })
  394. it('keeps pure Code Mode at one wire tool and exposes structured capture through the SDK only', async () => {
  395. const { ctx, parent, adapter } = await setup([
  396. toolCallResponse('c1', RUN_CODE_NAME, { code: 'return await tools.structured_output({ answer: 12 })' }),
  397. ], {
  398. toolMode: 'code',
  399. codeRun: async (request) => {
  400. const capture = request.bindings.at(0)?.functions[STRUCTURED_OUTPUT_TOOL]
  401. if (!capture) throw new Error('structured_output binding missing')
  402. await capture({ answer: 12 })
  403. return { logs: [], value: 'captured' }
  404. },
  405. })
  406. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  407. const result = await run.result
  408. expect(result.structured).toEqual({ answer: 12 })
  409. const request = adapter.requests[0]!
  410. expect(toolNames(request)).toEqual([RUN_CODE_NAME])
  411. expect(request.system).toContain('declare const tools:')
  412. expect(request.system).toContain('structured_output(args:')
  413. expect(request.system).toContain(STRUCTURED_OUTPUT_INSTRUCTION)
  414. await run.dispose()
  415. })
  416. it('discards a nested capture when the enclosing run_code execution fails', async () => {
  417. const { ctx, parent, adapter } = await setup([
  418. toolCallResponse('c1', RUN_CODE_NAME, { code: 'await tools.structured_output({ answer: 12 }); throw new Error("boom")' }),
  419. textResponse('outer code failed'),
  420. ], {
  421. toolMode: 'code',
  422. codeRun: async (request) => {
  423. const capture = request.bindings.at(0)?.functions[STRUCTURED_OUTPUT_TOOL]
  424. if (!capture) throw new Error('structured_output binding missing')
  425. await capture({ answer: 12 })
  426. return {
  427. logs: [],
  428. error: { kind: 'runtime', message: 'boom after capture' },
  429. } as never
  430. },
  431. })
  432. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  433. const result = await run.result
  434. expect(result.structured).toBeUndefined()
  435. expect(result.stopReason).toBe('error')
  436. expect(adapter.requests).toHaveLength(2)
  437. const child = ctx.agents.get(run.id)!
  438. const outer = child.session.events.find(event =>
  439. event.type === 'tool/result' && event.data.callId === CallId('c1'))
  440. expect(outer?.type === 'tool/result' && outer.data.isError).toBe(true)
  441. await run.dispose()
  442. })
  443. it('discards a nested capture when post-policy blocks the enclosing run_code result', async () => {
  444. const { ctx, parent, adapter } = await setup([
  445. toolCallResponse('c1', RUN_CODE_NAME, { code: 'return await tools.structured_output({ answer: 12 })' }),
  446. textResponse('outer code was blocked'),
  447. ], {
  448. toolMode: 'code',
  449. codeRun: async (request) => {
  450. const capture = request.bindings.at(0)?.functions[STRUCTURED_OUTPUT_TOOL]
  451. if (!capture) throw new Error('structured_output binding missing')
  452. await capture({ answer: 12 })
  453. return { logs: [], value: 'captured' }
  454. },
  455. })
  456. ctx.on('tools/post-execute', (exec, _result, next) => exec.name === RUN_CODE_NAME
  457. ? Promise.resolve({ kind: 'block' as const, feedback: [{ type: 'text' as const, text: 'outer blocked' }] })
  458. : next())
  459. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  460. const result = await run.result
  461. expect(result.structured).toBeUndefined()
  462. expect(result.stopReason).toBe('error')
  463. expect(adapter.requests).toHaveLength(2)
  464. await run.dispose()
  465. })
  466. it('the instruction rides ONLY structured requests: appended for the child, absent for a plain agent', async () => {
  467. const { ctx, parent, adapter } = await setup([
  468. textResponse('parent answer'),
  469. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 1 }),
  470. ])
  471. parent.send([{ type: 'text', text: 'hello' }])
  472. await parent.whenIdle()
  473. expect(adapter.requests[0]!.system ?? '').not.toContain(STRUCTURED_OUTPUT_INSTRUCTION)
  474. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  475. await run.result
  476. // The loop always assembles a base prompt (the harness identity section),
  477. // so the instruction APPENDS — never replaces.
  478. const childSystem = adapter.requests.at(-1)!.system!
  479. expect(childSystem.endsWith(STRUCTURED_OUTPUT_INSTRUCTION)).toBe(true)
  480. expect(childSystem.length).toBeGreaterThan(STRUCTURED_OUTPUT_INSTRUCTION.length)
  481. await run.dispose()
  482. })
  483. describe('scoped registration (each child owns its capture tool)', () => {
  484. it('a plain agent never sees the tool: nothing is registered globally at all', async () => {
  485. const { ctx, parent, adapter } = await setup([textResponse('parent answer')])
  486. parent.send([{ type: 'text', text: 'hello' }])
  487. await parent.whenIdle()
  488. // Scoped registration: the global view has no capture tool, ever.
  489. expect(ctx.tools.get(STRUCTURED_OUTPUT_TOOL)).toBeUndefined()
  490. expect(toolNames(adapter.requests[0]!)).not.toContain(STRUCTURED_OUTPUT_TOOL)
  491. })
  492. it('a structured child sees structured_output with ITS schema; a plain agent never sees the tool', async () => {
  493. const { ctx, parent, adapter } = await setup([
  494. // Parent turn (a plain agent): must NOT see the tool.
  495. textResponse('parent answer'),
  496. // Child turn: must see it, with the run's schema.
  497. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 42 }),
  498. ])
  499. parent.send([{ type: 'text', text: 'hello' }])
  500. await parent.whenIdle()
  501. expect(toolNames(adapter.requests[0]!)).not.toContain(STRUCTURED_OUTPUT_TOOL)
  502. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  503. await run.result
  504. const childRequest = adapter.requests[1]!
  505. expect(toolNames(childRequest)).toContain(STRUCTURED_OUTPUT_TOOL)
  506. const entry = childRequest.tools!.find(tool => tool.name === STRUCTURED_OUTPUT_TOOL)!
  507. expect(entry.parameters).toEqual(SCHEMA)
  508. await run.dispose()
  509. })
  510. it('two concurrent structured children each see their OWN schema', async () => {
  511. const otherSchema: StructuredOutputSchema = {
  512. type: 'object',
  513. properties: { verdict: { type: 'string', enum: ['real', 'bogus'] } },
  514. required: ['verdict'],
  515. }
  516. const { ctx, parent, adapter } = await setup([
  517. (options: GenerateOptions) => {
  518. // Answer with whatever schema this child was given — proves each
  519. // request carried the right one regardless of scheduling order.
  520. const entry = options.tools!.find(tool => tool.name === STRUCTURED_OUTPUT_TOOL)!
  521. const args = 'verdict' in (entry.parameters.properties as Record<string, unknown>)
  522. ? { verdict: 'real' }
  523. : { answer: 1 }
  524. return toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, args)
  525. },
  526. (options: GenerateOptions) => {
  527. const entry = options.tools!.find(tool => tool.name === STRUCTURED_OUTPUT_TOOL)!
  528. const args = 'verdict' in (entry.parameters.properties as Record<string, unknown>)
  529. ? { verdict: 'real' }
  530. : { answer: 1 }
  531. return toolCallResponse('c2', STRUCTURED_OUTPUT_TOOL, args)
  532. },
  533. ])
  534. const runA = await ctx.subagents.start('spawn', structuredRequest(parent))
  535. const runB = await ctx.subagents.start('spawn', structuredRequest(parent, { outputSchema: otherSchema }))
  536. const [a, b] = await Promise.all([runA.result, runB.result])
  537. expect(a.structured).toEqual({ answer: 1 })
  538. expect(b.structured).toEqual({ verdict: 'real' })
  539. const schemas = adapter.requests.map(request =>
  540. request.tools!.find(tool => tool.name === STRUCTURED_OUTPUT_TOOL)!.parameters)
  541. expect(schemas).toContainEqual(SCHEMA)
  542. expect(schemas).toContainEqual(otherSchema)
  543. await runA.dispose()
  544. await runB.dispose()
  545. })
  546. it('places the capture tool and instruction in their canonical orders', async () => {
  547. const { ctx, parent, adapter } = await setup([
  548. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 7 }),
  549. ])
  550. // A global tool sorts lexicographically after structured_output, while a
  551. // global section above the 190 band follows the capture instruction.
  552. ctx.tools.register({
  553. name: 'zz_probe',
  554. description: 'probe',
  555. parameters: { type: 'object', properties: {} },
  556. execute: () => Promise.resolve([{ type: 'text', text: 'x' }]),
  557. })
  558. ctx.systemPrompt.section({ name: 'after-band', order: 200, text: 'AFTER-BAND' })
  559. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  560. await run.result
  561. const request = adapter.requests[0]!
  562. const names = toolNames(request)
  563. expect(names.indexOf(STRUCTURED_OUTPUT_TOOL)).toBeGreaterThanOrEqual(0)
  564. expect(names.indexOf(STRUCTURED_OUTPUT_TOOL)).toBeLessThan(names.indexOf('zz_probe'))
  565. const system = request.system ?? ''
  566. const instructionAt = system.indexOf(STRUCTURED_OUTPUT_INSTRUCTION)
  567. expect(instructionAt).toBeGreaterThanOrEqual(0)
  568. expect(system.indexOf('AFTER-BAND')).toBeGreaterThan(instructionAt)
  569. await run.dispose()
  570. })
  571. it('a non-structured agent request keeps tools ABSENT when it had none (no tools: [] materialized)', async () => {
  572. const { parent, adapter } = await setup([textResponse('plain')])
  573. parent.send([{ type: 'text', text: 'q' }])
  574. await parent.whenIdle()
  575. const request = adapter.requests[0]!
  576. expect(request.tools).toBeUndefined()
  577. await new Promise(resolve => setTimeout(resolve, 0))
  578. })
  579. it('registrations ride the child fiber: disposing the run removes them; a provider reload mid-run cannot', async () => {
  580. const { ctx, parent, disposeProvider } = await setup([
  581. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 4 }),
  582. ])
  583. expect(ctx.tools.get(STRUCTURED_OUTPUT_TOOL)).toBeUndefined()
  584. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  585. // A backend hot-reload mid-run must not unregister the capture tool out
  586. // from under the live child: the registration rides the CHILD's fiber.
  587. disposeProvider()
  588. const result = await run.result
  589. expect(result.structured).toEqual({ answer: 4 })
  590. const child = ctx.agents.get(run.id)!
  591. expect(ctx.tools.get(STRUCTURED_OUTPUT_TOOL, child)).toBeDefined()
  592. await run.dispose()
  593. // Child disposed ⇒ its scoped registrations are gone.
  594. expect(ctx.tools.get(STRUCTURED_OUTPUT_TOOL, child)).toBeUndefined()
  595. })
  596. })
  597. it('a structured_output call from an agent WITHOUT a structured run is UNKNOWN_TOOL (the tool does not exist for it)', async () => {
  598. const { ctx, parent } = await setup([])
  599. const result = await ctx.tools.execute({
  600. callId: 'x' as never,
  601. name: STRUCTURED_OUTPUT_TOOL,
  602. arguments: { answer: 1 },
  603. agent: parent,
  604. })
  605. expect(result.isError).toBe(true)
  606. expect(result.error?.code).toBe('UNKNOWN_TOOL')
  607. })
  608. it('a structured_output call with NO calling agent at all is UNKNOWN_TOOL', async () => {
  609. const { ctx } = await setup([])
  610. const result = await ctx.tools.execute({
  611. callId: 'x' as never,
  612. name: STRUCTURED_OUTPUT_TOOL,
  613. arguments: { answer: 1 },
  614. })
  615. expect(result.isError).toBe(true)
  616. expect(result.error?.code).toBe('UNKNOWN_TOOL')
  617. })
  618. it('a failed execution stage is discarded and never promoted by a later call', async () => {
  619. const { ctx, parent } = await setup([
  620. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 1 }),
  621. ])
  622. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  623. // A prepended post-execute listener blocks the first capture without
  624. // delegating. The final-result notification discards that execution's
  625. // stage when it observes the error.
  626. let blocks = 1
  627. ctx.on('tools/post-execute', (exec, _result, next) => {
  628. if (exec.name === STRUCTURED_OUTPUT_TOOL && blocks > 0) {
  629. blocks -= 1
  630. return Promise.resolve({ kind: 'block' as const, feedback: [{ type: 'text' as const, text: 'rejected' }] })
  631. }
  632. return next()
  633. }, { prepend: true })
  634. const result = await run.result
  635. const child = ctx.agents.get(run.id)!
  636. // The blocked capture must NOT surface as structured success…
  637. expect(result.stopReason).toBe('error')
  638. expect(result.structured).toBeUndefined()
  639. // …and a LATER invalid call (its own body staged nothing) must not
  640. // resurrect c1's discarded value: drive the pipeline directly.
  641. const invalid = await ctx.tools.execute({
  642. callId: 'c2' as never,
  643. name: STRUCTURED_OUTPUT_TOOL,
  644. arguments: { answer: 'not-a-number' },
  645. agent: child,
  646. })
  647. expect(invalid.isError).toBe(true)
  648. // A fresh valid call still captures ITS OWN value.
  649. const valid = await ctx.tools.execute({
  650. callId: 'c3' as never,
  651. name: STRUCTURED_OUTPUT_TOOL,
  652. arguments: { answer: 9 },
  653. agent: child,
  654. })
  655. expect(valid.isError).toBeFalsy()
  656. await run.dispose()
  657. })
  658. it('reusing a failed execution\'s call id never promotes its discarded stage', async () => {
  659. const { ctx, parent } = await setup([
  660. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 1 }),
  661. ])
  662. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  663. // Block the first capture after its body stages a value. Its final error
  664. // discards that execution's stage.
  665. let blocks = 1
  666. ctx.on('tools/post-execute', (exec, _result, next) => {
  667. if (exec.name === STRUCTURED_OUTPUT_TOOL && blocks > 0) {
  668. blocks -= 1
  669. return Promise.resolve({ kind: 'block' as const, feedback: [{ type: 'text' as const, text: 'rejected' }] })
  670. }
  671. return next()
  672. }, { prepend: true })
  673. await run.result
  674. const child = ctx.agents.get(run.id)!
  675. // A SECOND capture call with the SAME call id whose body never stages
  676. // (invalid args throw before the stage): the discarded value must not ride
  677. // its acceptance.
  678. const reused = await ctx.tools.execute({
  679. callId: 'c1' as never,
  680. name: STRUCTURED_OUTPUT_TOOL,
  681. arguments: { answer: 'not-a-number' },
  682. agent: child,
  683. })
  684. expect(reused.isError).toBe(true)
  685. // Nothing was ever committed: a fresh valid call is still required.
  686. const valid = await ctx.tools.execute({
  687. callId: 'c1' as never,
  688. name: STRUCTURED_OUTPUT_TOOL,
  689. arguments: { answer: 5 },
  690. agent: child,
  691. })
  692. expect(valid.isError).toBeFalsy()
  693. await run.dispose()
  694. })
  695. it('a pre-execute deny with call-id reuse cannot promote another execution\'s stage', async () => {
  696. const { ctx, parent } = await setup([
  697. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 1 }),
  698. ])
  699. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  700. // Discard the first capture's stage via a final post-execute block.
  701. let blocks = 1
  702. ctx.on('tools/post-execute', (exec, _result, next) => {
  703. if (exec.name === STRUCTURED_OUTPUT_TOOL && blocks > 0) {
  704. blocks -= 1
  705. return Promise.resolve({ kind: 'block' as const, feedback: [{ type: 'text' as const, text: 'rejected' }] })
  706. }
  707. return next()
  708. }, { prepend: true })
  709. await run.result
  710. const child = ctx.agents.get(run.id)!
  711. // A prepended pre-execute deny skips the body, while the denied call still
  712. // reaches the final notification with the same adapter-minted call id.
  713. const offDeny = ctx.on('tools/pre-execute', (exec) => {
  714. if (exec.name === STRUCTURED_OUTPUT_TOOL) {
  715. return Promise.resolve({ kind: 'deny' as const, reason: 'outer veto' })
  716. }
  717. return undefined as never
  718. }, { prepend: true })
  719. const denied = await ctx.tools.execute({
  720. callId: 'c1' as never,
  721. name: STRUCTURED_OUTPUT_TOOL,
  722. arguments: { answer: 2 },
  723. agent: child,
  724. })
  725. expect(denied.isError).toBe(true)
  726. offDeny()
  727. // The discarded value was never promoted: a fresh valid call is required
  728. // (and succeeds, proving the runtime is not wedged).
  729. const valid = await ctx.tools.execute({
  730. callId: 'c1' as never,
  731. name: STRUCTURED_OUTPUT_TOOL,
  732. arguments: { answer: 5 },
  733. agent: child,
  734. })
  735. expect(valid.isError).toBeFalsy()
  736. await run.dispose()
  737. })
  738. })