structured.spec.ts 35 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764
  1. import { describe, expect, it } from 'vitest'
  2. import { Context } from '@deepseek-ai/cordis'
  3. import { createUserMessage, ToolCallId, type ContentBlock, type GenerateOptions } from '@deepseek-ai/dsh-llm'
  4. import { SessionId } from '@deepseek-ai/dsh-session'
  5. import AgentLoop from '@deepseek-ai/dsh-agent-loop'
  6. import { mountAgentLoopTestDependencies } from '@deepseek-ai/dsh-agent-loop-testkit'
  7. import InvariantRegistry from '@deepseek-ai/dsh-invariants'
  8. import type {} from '@deepseek-ai/dsh-system-prompt'
  9. import * as SessionInvariant from '@deepseek-ai/dsh-session/invariant'
  10. import * as AgentInvariant from '@deepseek-ai/dsh-agent/invariant'
  11. import * as AgentLoopInvariant from '@deepseek-ai/dsh-agent-loop/invariant'
  12. import SubagentRuntime, {
  13. type ResolvedSubagentStartRequest,
  14. type SubagentStartRequest,
  15. } from '@deepseek-ai/dsh-subagent'
  16. import SessionProjectionRegistry from '@deepseek-ai/dsh-session-projection'
  17. import type { Config as ToolConfig, ObjectJsonSchema } from '@deepseek-ai/dsh-tools'
  18. import { defineContentToolFixture, RUN_CODE_NAME } from '@deepseek-ai/dsh-tools'
  19. import { MockAdapter, textResponse, toolCallResponse } from '../../../core/agent-loop/tests/mock-adapter.ts'
  20. import { startInProcessRun } from '../src/index.ts'
  21. import {
  22. STRUCTURED_OUTPUT_INSTRUCTION,
  23. STRUCTURED_OUTPUT_TOOL,
  24. } from '../src/structured.ts'
  25. const testToolSignal = new AbortController().signal
  26. type Script = ConstructorParameters<typeof MockAdapter>[0]
  27. async function mountInvariants(ctx: Context): Promise<void> {
  28. await ctx.plugin(InvariantRegistry)
  29. await ctx.plugin(SessionInvariant)
  30. await ctx.plugin(AgentInvariant)
  31. await ctx.plugin(AgentLoopInvariant)
  32. }
  33. interface CodeRunRequestLike {
  34. bindings: { global: string; functions: Record<string, (args: unknown) => Promise<unknown>> }[]
  35. }
  36. interface SetupOptions {
  37. toolMode?: ToolConfig['mode']
  38. codeRun?: (request: CodeRunRequestLike) => Promise<{ logs: never[]; value?: unknown }>
  39. }
  40. const SCHEMA: ObjectJsonSchema = {
  41. type: 'object',
  42. properties: { answer: { type: 'number' }, note: { type: 'string' } },
  43. required: ['answer'],
  44. }
  45. /**
  46. * Real loop, scripted model, and inline fresh-conversation provider over the shared driver. Loading
  47. * spawn/fork here would create a dev-dependency cycle; their specs cover plugin integration while
  48. * this fixture isolates driver behavior and scripts the child's `structured_output` calls.
  49. */
  50. async function setup(script: Script, options: SetupOptions = {}) {
  51. const ctx = new Context()
  52. const adapter = new MockAdapter(script)
  53. await mountAgentLoopTestDependencies(ctx, {
  54. tools: { mode: options.toolMode ?? 'native' },
  55. })
  56. if (options.toolMode === 'ptc' || options.toolMode === 'both') {
  57. ctx.provide('codeRuntime', {
  58. language: 'typescript',
  59. isolation: 'test',
  60. run: options.codeRun ?? (() => Promise.resolve({ logs: [] })),
  61. } as never)
  62. }
  63. await mountInvariants(ctx)
  64. await ctx.plugin(AgentLoop, { agents: [] })
  65. await ctx.plugin(SessionProjectionRegistry)
  66. await ctx.plugin(SubagentRuntime)
  67. const disposeProvider = ctx.subagents.registerProvider({
  68. name: 'spawn',
  69. capabilities: { agentOptions: true, outputSchema: true, depthLimit: true, toolFilter: false, persona: false },
  70. inheritsParentContext: false,
  71. start: (request: ResolvedSubagentStartRequest) => startInProcessRun(request, {}),
  72. })
  73. ctx.llm.registerAdapter(['mock'], adapter)
  74. const parent = await ctx.agentLoop.create(SessionId('parent'), { provider: 'mock', model: 'mock' })
  75. return { ctx, parent, adapter, disposeProvider }
  76. }
  77. function structuredRequest(parent: SubagentStartRequest['parent'], extra?: Partial<SubagentStartRequest>): SubagentStartRequest {
  78. return {
  79. label: 'produce the answer',
  80. prompt: [{ type: 'text', text: 'produce the answer' }],
  81. parent,
  82. signal: new AbortController().signal,
  83. outputSchema: SCHEMA,
  84. ...extra,
  85. }
  86. }
  87. /** The tool names of one recorded model request. */
  88. function toolNames(request: GenerateOptions): string[] {
  89. return (request.tools ?? []).map(tool => tool.name)
  90. }
  91. describe('in-process structured output', () => {
  92. it('captures a valid structured_output call and surfaces result.structured', async () => {
  93. const { ctx, parent } = await setup([
  94. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 42, note: 'done' }),
  95. ])
  96. let acknowledgement: unknown
  97. ctx.on('tools/result', (exec, toolResult) => {
  98. if (exec.name === STRUCTURED_OUTPUT_TOOL && !toolResult.isError) acknowledgement = toolResult.value
  99. })
  100. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  101. const result = await run.result
  102. expect(result.stopReason).toBe('completed')
  103. expect(result.structured).toEqual({ answer: 42, note: 'done' })
  104. expect(acknowledgement).toEqual({ recorded: true })
  105. await run.dispose()
  106. })
  107. it('stops the turn after a successful capture — no extra model step is spent', async () => {
  108. const { ctx, parent, adapter } = await setup([
  109. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 1 }),
  110. textResponse('MUST NOT BE CONSUMED'),
  111. ])
  112. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  113. await run.result
  114. // The structured tool marks its successful result as turn-concluding.
  115. expect(adapter.requests.length).toBe(1)
  116. await run.dispose()
  117. })
  118. it('denies tool calls that FOLLOW the capture in the same response — terminal means terminal', async () => {
  119. // One model response carrying structured_output FIRST and a side-effecting
  120. // call after it: the continuation veto only fires at step end, so without
  121. // the pre-execute deny the trailing call would still run after the final
  122. // answer was accepted.
  123. const response = [
  124. ...toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 5 }).slice(0, -2),
  125. { type: 'block-start', index: 1, blockType: 'tool-call' },
  126. { type: 'block-end', index: 1, block: { type: 'tool-call', id: ToolCallId('c2'), name: 'side_effect', arguments: '{}' } },
  127. { type: 'usage', usage: { inputTokens: 10, outputTokens: 5 } },
  128. { type: 'finish', reason: { kind: 'tool-calls' } },
  129. ] as Script[number]
  130. const { ctx, parent } = await setup([response])
  131. let sideEffectRan = false
  132. ctx.tools.register(defineContentToolFixture({
  133. name: 'side_effect',
  134. description: 'probe',
  135. parameters: {},
  136. execute(): Promise<ContentBlock[]> {
  137. sideEffectRan = true
  138. return Promise.resolve([{ type: 'text', text: 'ran' }])
  139. },
  140. }))
  141. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  142. const result = await run.result
  143. expect(result.stopReason).toBe('completed')
  144. expect(result.structured).toEqual({ answer: 5 })
  145. // The deny skipped dispatch entirely: the probe body never ran.
  146. expect(sideEffectRan).toBe(false)
  147. await run.dispose()
  148. })
  149. it('a later prepended pre-execute listener cannot resurrect dispatch after capture', async () => {
  150. const response = [
  151. ...toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 5 }).slice(0, -2),
  152. { type: 'block-start', index: 1, blockType: 'tool-call' },
  153. { type: 'block-end', index: 1, block: { type: 'tool-call', id: ToolCallId('c2'), name: 'side_effect', arguments: '{}' } },
  154. { type: 'usage', usage: { inputTokens: 10, outputTokens: 5 } },
  155. { type: 'finish', reason: { kind: 'tool-calls' } },
  156. ] as Script[number]
  157. const { ctx, parent } = await setup([response])
  158. let sideEffectRan = false
  159. ctx.tools.register(defineContentToolFixture({
  160. name: 'side_effect',
  161. description: 'probe',
  162. parameters: {},
  163. execute(): Promise<ContentBlock[]> {
  164. sideEffectRan = true
  165. return Promise.resolve([{ type: 'text', text: 'ran' }])
  166. },
  167. }))
  168. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  169. // Registered after the child and prepended: this listener returns allow
  170. // after every downstream pre-execute decision. The service-owned guard
  171. // runs after the waterfall and can only deny, so the body still cannot run.
  172. ctx.on('tools/pre-execute', async (_exec, next) => {
  173. await next()
  174. return { kind: 'allow' as const }
  175. }, { prepend: true })
  176. const result = await run.result
  177. expect(result.structured).toEqual({ answer: 5 })
  178. expect(sideEffectRan).toBe(false)
  179. const child = ctx.agents.get(run.id)
  180. const sideEffectResult = child?.session.snapshotEvents().find(event =>
  181. event.type === 'tool/result' && event.data.message.source.callId === 'c2')
  182. expect(sideEffectResult?.type === 'tool/result' && sideEffectResult.data.message.content[0].isError).toBe(true)
  183. await run.dispose()
  184. })
  185. it('leaves tool calls that PRECEDE the capture in the same response untouched', async () => {
  186. const response = [
  187. { type: 'block-start', index: 0, blockType: 'tool-call' },
  188. { type: 'block-end', index: 0, block: { type: 'tool-call', id: ToolCallId('c1'), name: 'side_effect', arguments: '{}' } },
  189. ...toolCallResponse('c2', STRUCTURED_OUTPUT_TOOL, { answer: 6 }).map(chunk =>
  190. 'index' in chunk ? { ...chunk, index: 1 } : chunk),
  191. ] as Script[number]
  192. const { ctx, parent } = await setup([response])
  193. let sideEffectRan = false
  194. ctx.tools.register(defineContentToolFixture({
  195. name: 'side_effect',
  196. description: 'probe',
  197. parameters: {},
  198. execute(): Promise<ContentBlock[]> {
  199. sideEffectRan = true
  200. return Promise.resolve([{ type: 'text', text: 'ran' }])
  201. },
  202. }))
  203. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  204. const result = await run.result
  205. // The call ran BEFORE captured was set: the deny gate only guards the
  206. // window after the terminal answer landed.
  207. expect(sideEffectRan).toBe(true)
  208. expect(result.structured).toEqual({ answer: 6 })
  209. await run.dispose()
  210. })
  211. it('an invalid call gets an INVALID_ARGS isError result and the model retries in-turn', async () => {
  212. const { ctx, parent } = await setup([
  213. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 'not-a-number' }),
  214. toolCallResponse('c2', STRUCTURED_OUTPUT_TOOL, { answer: 7 }),
  215. ])
  216. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  217. const result = await run.result
  218. expect(result.structured).toEqual({ answer: 7 })
  219. expect(result.stopReason).toBe('completed')
  220. // The child's log carries the isError tool/result for the invalid call.
  221. const child = ctx.agents.get(run.id)!
  222. const results = child.session.snapshotEvents().filter(e => e.type === 'tool/result')
  223. expect(results.length).toBe(2)
  224. expect(results[0]!.data.message.content[0].isError).toBe(true)
  225. await run.dispose()
  226. })
  227. it('a clean finish without a capture is an immediate error to the parent — deliberately NO re-prompt', async () => {
  228. const { ctx, parent, adapter } = await setup([
  229. textResponse('here is my answer in prose'),
  230. textResponse('MUST NOT BE CONSUMED'),
  231. ])
  232. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  233. const result = await run.result
  234. expect(result.stopReason).toBe('error')
  235. expect(result.structured).toBeUndefined()
  236. // Exactly one model request and one caller-supplied user message: no nudge turn exists.
  237. expect(adapter.requests.length).toBe(1)
  238. const child = ctx.agents.get(run.id)!
  239. expect(child.session.snapshotEvents().filter(e => e.type === 'user/message' && e.data.source.kind !== 'plugin').length).toBe(1)
  240. await run.dispose()
  241. })
  242. it('an errored child keeps its honest error result (no capture expected)', async () => {
  243. // Script exhaustion on the first call → the child turn errors.
  244. const { ctx, parent, adapter } = await setup([])
  245. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  246. const result = await run.result
  247. expect(result.stopReason).toBe('error')
  248. expect(adapter.requests.length).toBe(1)
  249. await run.dispose()
  250. })
  251. it('a cancel landing after a clean capture-less turn settles aborted, not error', async () => {
  252. const { ctx, parent } = await setup([textResponse('prose, no capture')])
  253. const controller = new AbortController()
  254. const run = await ctx.subagents.start('spawn', structuredRequest(parent, { signal: controller.signal }))
  255. // Cancel synchronously inside the turn's end recording: the cancel
  256. // contract outranks the schema shortfall, so the result maps to aborted.
  257. ctx.on('session/event', (session, event) => {
  258. const child = ctx.agents.get(run.id)
  259. if (session === child?.session && event.type === 'turn/end') controller.abort('cancelled at turn end')
  260. })
  261. const result = await run.result
  262. expect(result.stopReason).toBe('aborted')
  263. await run.dispose()
  264. })
  265. it('rejects a schema outside the subset loud, before any child exists', async () => {
  266. const { ctx, parent } = await setup([])
  267. await expect(ctx.subagents.start('spawn', structuredRequest(parent, {
  268. outputSchema: { type: 'object', oneOf: [] } as unknown as ObjectJsonSchema,
  269. }))).rejects.toThrow(/unsupported JSON schema/)
  270. expect(ctx.agents.get(SessionId('parent'))).toBeDefined()
  271. })
  272. it('a schema carrying non-JSON values fails as JsonSchemaError at the validation boundary', async () => {
  273. const { ctx, parent } = await setup([])
  274. // Semantic assertion runs before provider startup.
  275. await expect(ctx.subagents.start('spawn', structuredRequest(parent, {
  276. outputSchema: { type: 'object', default: () => {} } as unknown as ObjectJsonSchema,
  277. }))).rejects.toThrow(/unsupported JSON schema.*annotation must be lossless JSON data/)
  278. })
  279. it('a post-execute BLOCK on the capture call denies the capture: log and result agree on failure', async () => {
  280. const { ctx, parent, adapter } = await setup([
  281. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 7 }),
  282. textResponse('continues after the blocked capture'),
  283. ])
  284. // A PostToolUse-style hook turns the tool body's provisional success into
  285. // the authoritative final error observed by the commit notification.
  286. ctx.on('tools/post-execute', (exec, _result, next) => {
  287. if (exec.name === STRUCTURED_OUTPUT_TOOL) {
  288. return Promise.resolve({ kind: 'block' as const, feedback: [{ type: 'text' as const, text: 'capture rejected by hook' }] })
  289. }
  290. return next()
  291. })
  292. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  293. const result = await run.result
  294. // No capture was committed: the run reports the schema shortfall...
  295. expect(result.structured).toBeUndefined()
  296. expect(result.stopReason).toBe('error')
  297. // ...the logged tool result is the blocked isError with the feedback...
  298. const child = ctx.agents.get(run.id)!
  299. const results = child.session.snapshotEvents().filter(e => e.type === 'tool/result')
  300. expect(results[0]!.data.message.content[0].isError).toBe(true)
  301. expect(JSON.stringify(results[0]!.data.message.content)).toContain('capture rejected by hook')
  302. // ...and the turn CONTINUED past the blocked call (no captured veto):
  303. // the model got to react to the failure with a second step.
  304. expect(adapter.requests.length).toBe(2)
  305. await run.dispose()
  306. })
  307. it('a post-execute accept-with-replacement still commits the capture', async () => {
  308. const { ctx, parent } = await setup([
  309. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 8 }),
  310. ])
  311. ctx.on('tools/post-execute', (exec, _result, next) => {
  312. if (exec.name === STRUCTURED_OUTPUT_TOOL) {
  313. return Promise.resolve({ kind: 'accept' as const, content: [{ type: 'text' as const, text: 'recorded (rewritten)' }] })
  314. }
  315. return next()
  316. })
  317. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  318. const result = await run.result
  319. expect(result.stopReason).toBe('completed')
  320. expect(result.structured).toEqual({ answer: 8 })
  321. await run.dispose()
  322. })
  323. it('commits only after a later prepended post-execute wrapper returns the authoritative result', async () => {
  324. const { ctx, parent } = await setup([
  325. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 8 }),
  326. textResponse('capture was rejected'),
  327. ])
  328. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  329. // Registered after attachment and prepended, so it wraps every listener
  330. // the child installed. It delegates first, then converts the apparent
  331. // capture success into the pipeline's authoritative failure.
  332. ctx.on('tools/post-execute', async (exec, _result, next) => {
  333. const downstream = await next()
  334. if (exec.name !== STRUCTURED_OUTPUT_TOOL) return downstream
  335. return { kind: 'block' as const, feedback: [{ type: 'text' as const, text: 'rejected after downstream' }] }
  336. }, { prepend: true })
  337. const result = await run.result
  338. expect(result.structured).toBeUndefined()
  339. expect(result.stopReason).toBe('error')
  340. const child = ctx.agents.get(run.id)
  341. const captureResult = child?.session.snapshotEvents().find(event =>
  342. event.type === 'tool/result' && event.data.message.source.callId === 'c1')
  343. expect(captureResult?.type === 'tool/result' && captureResult.data.message.content[0].isError).toBe(true)
  344. await run.dispose()
  345. })
  346. it('appends the structured instruction to the child REQUEST\'s system text (base prompt preserved)', async () => {
  347. const { ctx, parent, adapter } = await setup([toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 1 })])
  348. // A context-wide section stands in for the deployment persona: the
  349. // instruction must APPEND to the other scoped and global sections, not
  350. // replace them (AgentOptions has no prompt field — the instruction is an
  351. // ordinary child-scoped prompt registration).
  352. ctx.systemPrompt.section({ name: 'test:persona', order: 10, text: 'You are a counter.' })
  353. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  354. await run.result
  355. const childRequest = adapter.requests.at(-1)!
  356. expect(childRequest.system).toContain('You are a counter.')
  357. expect(childRequest.system!.endsWith(STRUCTURED_OUTPUT_INSTRUCTION)).toBe(true)
  358. expect(childRequest.system!.indexOf(STRUCTURED_OUTPUT_INSTRUCTION)).toBeGreaterThan(0)
  359. await run.dispose()
  360. })
  361. it('keeps pure PTC mode at one wire tool and exposes structured capture through the SDK only', async () => {
  362. const { ctx, parent, adapter } = await setup([
  363. toolCallResponse('c1', RUN_CODE_NAME, { code: 'return await tools.structured_output({ answer: 12 })', description: 'Capture the structured answer' }),
  364. ], {
  365. toolMode: 'ptc',
  366. codeRun: async (request) => {
  367. const capture = request.bindings.at(0)?.functions[STRUCTURED_OUTPUT_TOOL]
  368. if (!capture) throw new Error('structured_output binding missing')
  369. await capture({ answer: 12 })
  370. return { logs: [], value: 'captured' }
  371. },
  372. })
  373. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  374. const result = await run.result
  375. expect(result.structured).toEqual({ answer: 12 })
  376. const request = adapter.requests[0]!
  377. expect(toolNames(request)).toEqual([RUN_CODE_NAME])
  378. expect(request.system).toContain('interface ToolArgsMap')
  379. expect(request.system).toContain('interface ToolOutputMap')
  380. expect(request.system).toContain('recorded: true;')
  381. expect(request.system).toContain('Promise<ToolOutputMap[K]>')
  382. expect(request.system).toContain(STRUCTURED_OUTPUT_INSTRUCTION)
  383. await run.dispose()
  384. })
  385. it('discards a nested capture when the enclosing run_code execution fails', async () => {
  386. const { ctx, parent, adapter } = await setup([
  387. toolCallResponse('c1', RUN_CODE_NAME, { code: 'await tools.structured_output({ answer: 12 }); throw new Error("boom")', description: 'Capture then fail the program' }),
  388. textResponse('outer code failed'),
  389. ], {
  390. toolMode: 'ptc',
  391. codeRun: async (request) => {
  392. const capture = request.bindings.at(0)?.functions[STRUCTURED_OUTPUT_TOOL]
  393. if (!capture) throw new Error('structured_output binding missing')
  394. await capture({ answer: 12 })
  395. return {
  396. logs: [],
  397. error: { kind: 'runtime', message: 'boom after capture' },
  398. } as never
  399. },
  400. })
  401. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  402. const result = await run.result
  403. expect(result.structured).toBeUndefined()
  404. expect(result.stopReason).toBe('error')
  405. expect(adapter.requests).toHaveLength(2)
  406. const child = ctx.agents.get(run.id)!
  407. const outer = child.session.snapshotEvents().find(event =>
  408. event.type === 'tool/result' && event.data.message.source.callId === ToolCallId('c1'))
  409. expect(outer?.type === 'tool/result' && outer.data.message.content[0].isError).toBe(true)
  410. await run.dispose()
  411. })
  412. it('discards a nested capture when post-policy blocks the enclosing run_code result', async () => {
  413. const { ctx, parent, adapter } = await setup([
  414. toolCallResponse('c1', RUN_CODE_NAME, { code: 'return await tools.structured_output({ answer: 12 })', description: 'Capture the structured answer' }),
  415. textResponse('outer code was blocked'),
  416. ], {
  417. toolMode: 'ptc',
  418. codeRun: async (request) => {
  419. const capture = request.bindings.at(0)?.functions[STRUCTURED_OUTPUT_TOOL]
  420. if (!capture) throw new Error('structured_output binding missing')
  421. await capture({ answer: 12 })
  422. return { logs: [], value: 'captured' }
  423. },
  424. })
  425. ctx.on('tools/post-execute', (exec, _result, next) => exec.name === RUN_CODE_NAME
  426. ? Promise.resolve({ kind: 'block' as const, feedback: [{ type: 'text' as const, text: 'outer blocked' }] })
  427. : next())
  428. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  429. const result = await run.result
  430. expect(result.structured).toBeUndefined()
  431. expect(result.stopReason).toBe('error')
  432. expect(adapter.requests).toHaveLength(2)
  433. await run.dispose()
  434. })
  435. it('the instruction rides ONLY structured requests: appended for the child, absent for a plain agent', async () => {
  436. const { ctx, parent, adapter } = await setup([
  437. textResponse('parent answer'),
  438. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 1 }),
  439. ])
  440. parent.followup(createUserMessage({ content: [{ type: 'text', text: 'hello' }], source: { kind: 'user' } }))
  441. await parent.whenIdle()
  442. expect(adapter.requests[0]!.system ?? '').not.toContain(STRUCTURED_OUTPUT_INSTRUCTION)
  443. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  444. await run.result
  445. // The loop always assembles a base prompt (the harness identity section),
  446. // so the instruction APPENDS — never replaces.
  447. const childSystem = adapter.requests.at(-1)!.system!
  448. expect(childSystem.endsWith(STRUCTURED_OUTPUT_INSTRUCTION)).toBe(true)
  449. expect(childSystem.length).toBeGreaterThan(STRUCTURED_OUTPUT_INSTRUCTION.length)
  450. await run.dispose()
  451. })
  452. describe('scoped registration (each child owns its capture tool)', () => {
  453. it('a plain agent never sees the tool: nothing is registered globally at all', async () => {
  454. const { ctx, parent, adapter } = await setup([textResponse('parent answer')])
  455. parent.followup(createUserMessage({ content: [{ type: 'text', text: 'hello' }], source: { kind: 'user' } }))
  456. await parent.whenIdle()
  457. // Scoped registration: the global view has no capture tool, ever.
  458. expect(ctx.tools.get(STRUCTURED_OUTPUT_TOOL)).toBeUndefined()
  459. expect(toolNames(adapter.requests[0]!)).not.toContain(STRUCTURED_OUTPUT_TOOL)
  460. })
  461. it('a structured child sees structured_output with ITS schema; a plain agent never sees the tool', async () => {
  462. const { ctx, parent, adapter } = await setup([
  463. // Parent turn (a plain agent): must NOT see the tool.
  464. textResponse('parent answer'),
  465. // Child turn: must see it, with the run's schema.
  466. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 42 }),
  467. ])
  468. parent.followup(createUserMessage({ content: [{ type: 'text', text: 'hello' }], source: { kind: 'user' } }))
  469. await parent.whenIdle()
  470. expect(toolNames(adapter.requests[0]!)).not.toContain(STRUCTURED_OUTPUT_TOOL)
  471. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  472. await run.result
  473. const childRequest = adapter.requests[1]!
  474. expect(toolNames(childRequest)).toContain(STRUCTURED_OUTPUT_TOOL)
  475. const entry = childRequest.tools!.find(tool => tool.name === STRUCTURED_OUTPUT_TOOL)!
  476. expect(entry.parameters).toEqual(SCHEMA)
  477. await run.dispose()
  478. })
  479. it('two concurrent structured children each see their OWN schema', async () => {
  480. const otherSchema: ObjectJsonSchema = {
  481. type: 'object',
  482. properties: { verdict: { type: 'string', enum: ['real', 'bogus'] } },
  483. required: ['verdict'],
  484. }
  485. const { ctx, parent, adapter } = await setup([
  486. (options: GenerateOptions) => {
  487. // Answer with whatever schema this child was given — proves each
  488. // request carried the right one regardless of scheduling order.
  489. const entry = options.tools!.find(tool => tool.name === STRUCTURED_OUTPUT_TOOL)!
  490. const args = 'verdict' in (entry.parameters.properties as Record<string, unknown>)
  491. ? { verdict: 'real' }
  492. : { answer: 1 }
  493. return toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, args)
  494. },
  495. (options: GenerateOptions) => {
  496. const entry = options.tools!.find(tool => tool.name === STRUCTURED_OUTPUT_TOOL)!
  497. const args = 'verdict' in (entry.parameters.properties as Record<string, unknown>)
  498. ? { verdict: 'real' }
  499. : { answer: 1 }
  500. return toolCallResponse('c2', STRUCTURED_OUTPUT_TOOL, args)
  501. },
  502. ])
  503. const runA = await ctx.subagents.start('spawn', structuredRequest(parent))
  504. const runB = await ctx.subagents.start('spawn', structuredRequest(parent, { outputSchema: otherSchema }))
  505. const [a, b] = await Promise.all([runA.result, runB.result])
  506. expect(a.structured).toEqual({ answer: 1 })
  507. expect(b.structured).toEqual({ verdict: 'real' })
  508. const schemas = adapter.requests.map(request =>
  509. request.tools!.find(tool => tool.name === STRUCTURED_OUTPUT_TOOL)!.parameters)
  510. expect(schemas).toContainEqual(SCHEMA)
  511. expect(schemas).toContainEqual(otherSchema)
  512. await runA.dispose()
  513. await runB.dispose()
  514. })
  515. it('places the capture tool and instruction in their canonical orders', async () => {
  516. const { ctx, parent, adapter } = await setup([
  517. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 7 }),
  518. ])
  519. // A global tool sorts lexicographically after structured_output, while a
  520. // global section after the final-output slot follows the capture instruction.
  521. ctx.tools.register(defineContentToolFixture({
  522. name: 'zz_probe',
  523. description: 'probe',
  524. parameters: {},
  525. execute: () => Promise.resolve([{ type: 'text', text: 'x' }]),
  526. }))
  527. ctx.systemPrompt.section({
  528. name: 'after-band',
  529. order: ctx.systemPrompt.getSectionOrder('STRUCTURED_OUTPUT') + 10,
  530. text: 'AFTER-BAND',
  531. })
  532. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  533. await run.result
  534. const request = adapter.requests[0]!
  535. const names = toolNames(request)
  536. expect(names.indexOf(STRUCTURED_OUTPUT_TOOL)).toBeGreaterThanOrEqual(0)
  537. expect(names.indexOf(STRUCTURED_OUTPUT_TOOL)).toBeLessThan(names.indexOf('zz_probe'))
  538. const system = request.system ?? ''
  539. const instructionAt = system.indexOf(STRUCTURED_OUTPUT_INSTRUCTION)
  540. expect(instructionAt).toBeGreaterThanOrEqual(0)
  541. expect(system.indexOf('AFTER-BAND')).toBeGreaterThan(instructionAt)
  542. await run.dispose()
  543. })
  544. it('a non-structured agent request keeps tools ABSENT when it had none (no tools: [] materialized)', async () => {
  545. const { parent, adapter } = await setup([textResponse('plain')])
  546. parent.followup(createUserMessage({ content: [{ type: 'text', text: 'q' }], source: { kind: 'user' } }))
  547. await parent.whenIdle()
  548. const request = adapter.requests[0]!
  549. expect(request.tools).toBeUndefined()
  550. await new Promise(resolve => setTimeout(resolve, 0))
  551. })
  552. it('registrations ride the child fiber: disposing the run removes them; a provider reload mid-run cannot', async () => {
  553. const { ctx, parent, disposeProvider } = await setup([
  554. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 4 }),
  555. ])
  556. expect(ctx.tools.get(STRUCTURED_OUTPUT_TOOL)).toBeUndefined()
  557. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  558. // A backend hot-reload mid-run must not unregister the capture tool out
  559. // from under the live child: the registration rides the CHILD's fiber.
  560. disposeProvider()
  561. const result = await run.result
  562. expect(result.structured).toEqual({ answer: 4 })
  563. const child = ctx.agents.get(run.id)!
  564. expect(ctx.tools.get(STRUCTURED_OUTPUT_TOOL, child)).toBeDefined()
  565. await run.dispose()
  566. // Child disposed ⇒ its scoped registrations are gone.
  567. expect(ctx.tools.get(STRUCTURED_OUTPUT_TOOL, child)).toBeUndefined()
  568. })
  569. })
  570. it('a structured_output call from an agent WITHOUT a structured run is UNKNOWN_TOOL (the tool does not exist for it)', async () => {
  571. const { ctx, parent } = await setup([])
  572. const result = await ctx.tools.execute({
  573. signal: testToolSignal,
  574. callId: 'x' as never,
  575. name: STRUCTURED_OUTPUT_TOOL,
  576. arguments: { answer: 1 },
  577. agent: parent,
  578. })
  579. expect(result.isError).toBe(true)
  580. expect(result.error?.info?.code).toBe('UNKNOWN_TOOL')
  581. })
  582. it('a structured_output call with NO calling agent at all is UNKNOWN_TOOL', async () => {
  583. const { ctx } = await setup([])
  584. const result = await ctx.tools.execute({
  585. signal: testToolSignal,
  586. callId: 'x' as never,
  587. name: STRUCTURED_OUTPUT_TOOL,
  588. arguments: { answer: 1 },
  589. })
  590. expect(result.isError).toBe(true)
  591. expect(result.error?.info?.code).toBe('UNKNOWN_TOOL')
  592. })
  593. it('a failed execution stage is discarded and never promoted by a later call', async () => {
  594. const { ctx, parent } = await setup([
  595. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 1 }),
  596. ])
  597. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  598. // A prepended post-execute listener blocks the first capture without
  599. // delegating. The final-result notification discards that execution's
  600. // stage when it observes the error.
  601. let blocks = 1
  602. ctx.on('tools/post-execute', (exec, _result, next) => {
  603. if (exec.name === STRUCTURED_OUTPUT_TOOL && blocks > 0) {
  604. blocks -= 1
  605. return Promise.resolve({ kind: 'block' as const, feedback: [{ type: 'text' as const, text: 'rejected' }] })
  606. }
  607. return next()
  608. }, { prepend: true })
  609. const result = await run.result
  610. const child = ctx.agents.get(run.id)!
  611. // The blocked capture must NOT surface as structured success…
  612. expect(result.stopReason).toBe('error')
  613. expect(result.structured).toBeUndefined()
  614. // …and a LATER invalid call (its own body staged nothing) must not
  615. // resurrect c1's discarded value: drive the pipeline directly.
  616. const invalid = await ctx.tools.execute({
  617. signal: testToolSignal,
  618. callId: 'c2' as never,
  619. name: STRUCTURED_OUTPUT_TOOL,
  620. arguments: { answer: 'not-a-number' },
  621. agent: child,
  622. })
  623. expect(invalid.isError).toBe(true)
  624. // A fresh valid call still captures ITS OWN value.
  625. const valid = await ctx.tools.execute({
  626. signal: testToolSignal,
  627. callId: 'c3' as never,
  628. name: STRUCTURED_OUTPUT_TOOL,
  629. arguments: { answer: 9 },
  630. agent: child,
  631. })
  632. expect(valid.isError).toBeFalsy()
  633. await run.dispose()
  634. })
  635. it('reusing a failed execution\'s call id never promotes its discarded stage', async () => {
  636. const { ctx, parent } = await setup([
  637. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 1 }),
  638. ])
  639. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  640. // Block the first capture after its body stages a value. Its final error
  641. // discards that execution's stage.
  642. let blocks = 1
  643. ctx.on('tools/post-execute', (exec, _result, next) => {
  644. if (exec.name === STRUCTURED_OUTPUT_TOOL && blocks > 0) {
  645. blocks -= 1
  646. return Promise.resolve({ kind: 'block' as const, feedback: [{ type: 'text' as const, text: 'rejected' }] })
  647. }
  648. return next()
  649. }, { prepend: true })
  650. await run.result
  651. const child = ctx.agents.get(run.id)!
  652. // A SECOND capture call with the SAME call id whose body never stages
  653. // (invalid args throw before the stage): the discarded value must not ride
  654. // its acceptance.
  655. const reused = await ctx.tools.execute({
  656. signal: testToolSignal,
  657. callId: 'c1' as never,
  658. name: STRUCTURED_OUTPUT_TOOL,
  659. arguments: { answer: 'not-a-number' },
  660. agent: child,
  661. })
  662. expect(reused.isError).toBe(true)
  663. // Nothing was ever committed: a fresh valid call is still required.
  664. const valid = await ctx.tools.execute({
  665. signal: testToolSignal,
  666. callId: 'c1' as never,
  667. name: STRUCTURED_OUTPUT_TOOL,
  668. arguments: { answer: 5 },
  669. agent: child,
  670. })
  671. expect(valid.isError).toBeFalsy()
  672. await run.dispose()
  673. })
  674. it('a pre-execute deny with call-id reuse cannot promote another execution\'s stage', async () => {
  675. const { ctx, parent } = await setup([
  676. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 1 }),
  677. ])
  678. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  679. // Discard the first capture's stage via a final post-execute block.
  680. let blocks = 1
  681. ctx.on('tools/post-execute', (exec, _result, next) => {
  682. if (exec.name === STRUCTURED_OUTPUT_TOOL && blocks > 0) {
  683. blocks -= 1
  684. return Promise.resolve({ kind: 'block' as const, feedback: [{ type: 'text' as const, text: 'rejected' }] })
  685. }
  686. return next()
  687. }, { prepend: true })
  688. await run.result
  689. const child = ctx.agents.get(run.id)!
  690. // A prepended pre-execute deny skips the body, while the denied call still
  691. // reaches the final notification with the same adapter-minted call id.
  692. const offDeny = ctx.on('tools/pre-execute', (exec) => {
  693. if (exec.name === STRUCTURED_OUTPUT_TOOL) {
  694. return Promise.resolve({ kind: 'deny' as const, reason: 'outer veto' })
  695. }
  696. return undefined as never
  697. }, { prepend: true })
  698. const denied = await ctx.tools.execute({
  699. signal: testToolSignal,
  700. callId: 'c1' as never,
  701. name: STRUCTURED_OUTPUT_TOOL,
  702. arguments: { answer: 2 },
  703. agent: child,
  704. })
  705. expect(denied.isError).toBe(true)
  706. offDeny()
  707. // The discarded value was never promoted: a fresh valid call is required
  708. // (and succeeds, proving the runtime is not wedged).
  709. const valid = await ctx.tools.execute({
  710. signal: testToolSignal,
  711. callId: 'c1' as never,
  712. name: STRUCTURED_OUTPUT_TOOL,
  713. arguments: { answer: 5 },
  714. agent: child,
  715. })
  716. expect(valid.isError).toBeFalsy()
  717. await run.dispose()
  718. })
  719. })