structured.spec.ts 36 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780
  1. import { describe, expect, it } from 'vitest'
  2. import { Context } from 'cordis'
  3. import { createUserMessage, CallId, type ContentBlock, type GenerateOptions } from '@deepseek-ai/dsh-llm'
  4. import { SessionId } from '@deepseek-ai/dsh-session'
  5. import AgentLoop from '@deepseek-ai/dsh-agent-loop'
  6. import { mountAgentLoopTestDependencies } from '@deepseek-ai/dsh-agent-loop-testkit'
  7. import InvariantService from '@deepseek-ai/dsh-invariants'
  8. import * as SessionInvariant from '@deepseek-ai/dsh-session/invariant'
  9. import * as AgentInvariant from '@deepseek-ai/dsh-agent/invariant'
  10. import * as AgentLoopInvariant from '@deepseek-ai/dsh-agent-loop/invariant'
  11. import SubagentService, { type SubagentStartRequest } from '@deepseek-ai/dsh-subagent'
  12. import type { Config as ToolConfig, ObjectJsonSchema } from '@deepseek-ai/dsh-tools'
  13. import { defineContentToolFixture, RUN_CODE_NAME } from '@deepseek-ai/dsh-tools'
  14. import { MockAdapter, textResponse, toolCallResponse } from '../../../core/agent-loop/tests/mock-adapter.ts'
  15. import { startInProcessRun } from '../src/index.ts'
  16. import {
  17. STRUCTURED_OUTPUT_INSTRUCTION,
  18. STRUCTURED_OUTPUT_TOOL,
  19. } from '../src/structured.ts'
  20. const testToolSignal = new AbortController().signal
  21. type Script = ConstructorParameters<typeof MockAdapter>[0]
  22. async function mountInvariants(ctx: Context): Promise<void> {
  23. await ctx.plugin(InvariantService)
  24. await ctx.plugin(SessionInvariant)
  25. await ctx.plugin(AgentInvariant)
  26. await ctx.plugin(AgentLoopInvariant)
  27. }
  28. interface CodeRunRequestLike {
  29. bindings: { global: string; functions: Record<string, (args: unknown) => Promise<unknown>> }[]
  30. }
  31. interface SetupOptions {
  32. toolMode?: ToolConfig['mode']
  33. codeRun?: (request: CodeRunRequestLike) => Promise<{ logs: never[]; value?: unknown }>
  34. }
  35. const SCHEMA: ObjectJsonSchema = {
  36. type: 'object',
  37. properties: { answer: { type: 'number' }, note: { type: 'string' } },
  38. required: ['answer'],
  39. }
  40. /**
  41. * Real loop, scripted model, and inline fresh-conversation provider over the shared driver. Loading
  42. * spawn/fork here would create a dev-dependency cycle; their specs cover plugin integration while
  43. * this fixture isolates driver behavior and scripts the child's `structured_output` calls.
  44. */
  45. async function setup(script: Script, options: SetupOptions = {}) {
  46. const ctx = new Context()
  47. const adapter = new MockAdapter(script)
  48. await mountAgentLoopTestDependencies(ctx, {
  49. tools: { mode: options.toolMode ?? 'native' },
  50. })
  51. if (options.toolMode === 'code' || options.toolMode === 'both') {
  52. ctx.provide('codeRuntime', {
  53. language: 'typescript',
  54. isolation: 'test',
  55. run: options.codeRun ?? (() => Promise.resolve({ logs: [] })),
  56. } as never)
  57. }
  58. await mountInvariants(ctx)
  59. await ctx.plugin(AgentLoop, { agents: [] })
  60. await ctx.plugin(SubagentService)
  61. const disposeProvider = ctx.subagents.registerProvider({
  62. name: 'spawn',
  63. capabilities: { outputSchema: true, depthLimit: true, toolFilter: false, persona: false },
  64. inheritsParentContext: false,
  65. start: (request: SubagentStartRequest) => startInProcessRun(request, {}),
  66. })
  67. ctx.llm.registerAdapter(['mock'], adapter)
  68. const parent = ctx.agentLoop.create(SessionId('parent'), { provider: 'mock', model: 'mock' })
  69. return { ctx, parent, adapter, disposeProvider }
  70. }
  71. function structuredRequest(parent: SubagentStartRequest['parent'], extra?: Partial<SubagentStartRequest>): SubagentStartRequest {
  72. return {
  73. prompt: [{ type: 'text', text: 'produce the answer' }],
  74. parent,
  75. signal: new AbortController().signal,
  76. outputSchema: SCHEMA,
  77. ...extra,
  78. }
  79. }
  80. /** The tool names of one recorded model request. */
  81. function toolNames(request: GenerateOptions): string[] {
  82. return (request.tools ?? []).map(tool => tool.name)
  83. }
  84. describe('in-process structured output', () => {
  85. it('captures a valid structured_output call and surfaces result.structured', async () => {
  86. const { ctx, parent } = await setup([
  87. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 42, note: 'done' }),
  88. ])
  89. let acknowledgement: unknown
  90. ctx.on('tools/result', (exec, toolResult) => {
  91. if (exec.name === STRUCTURED_OUTPUT_TOOL && !toolResult.isError) acknowledgement = toolResult.value
  92. })
  93. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  94. const result = await run.result
  95. expect(result.stopReason).toBe('completed')
  96. expect(result.structured).toEqual({ answer: 42, note: 'done' })
  97. expect(acknowledgement).toEqual({ recorded: true })
  98. await run.dispose()
  99. })
  100. it('stops the turn after a successful capture — no extra model step is spent', async () => {
  101. const { ctx, parent, adapter } = await setup([
  102. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 1 }),
  103. textResponse('MUST NOT BE CONSUMED'),
  104. ])
  105. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  106. await run.result
  107. // The structured tool marks its successful result as turn-concluding.
  108. expect(adapter.requests.length).toBe(1)
  109. await run.dispose()
  110. })
  111. it('strict steer rejects delivery once the structured result is captured', async () => {
  112. // Hold the capture's tool result open so the child is observably running
  113. // with a committed capture: the pending agent/turn-stop checkpoint is
  114. // terminal, and the loop would DISCARD a steering message, so an
  115. // acknowledged delivery here would be a lie.
  116. let releaseResult: (() => void) | undefined
  117. const { ctx, parent } = await setup([
  118. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 7 }),
  119. ])
  120. ctx.on('agent/post-step', (agent) => {
  121. if (agent.session.header.parentSession === undefined || releaseResult !== undefined) return
  122. return new Promise<void>((resolve) => { releaseResult = resolve })
  123. })
  124. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  125. await new Promise<void>((resolve) => {
  126. const timer = setInterval(() => {
  127. if (releaseResult !== undefined) { clearInterval(timer); resolve() }
  128. }, 5)
  129. })
  130. expect(() => { run.steer!([{ type: 'text', text: 'one more thing' }]) })
  131. .toThrow(/already reported its structured result; the message was not delivered/)
  132. releaseResult!()
  133. const result = await run.result
  134. expect(result.structured).toEqual({ answer: 7 })
  135. await run.dispose()
  136. })
  137. it('denies tool calls that FOLLOW the capture in the same response — terminal means terminal', async () => {
  138. // One model response carrying structured_output FIRST and a side-effecting
  139. // call after it: the continuation veto only fires at step end, so without
  140. // the pre-execute deny the trailing call would still run after the final
  141. // answer was accepted.
  142. const response = [
  143. ...toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 5 }).slice(0, -2),
  144. { type: 'block-start', index: 1, blockType: 'tool-call' },
  145. { type: 'block-end', index: 1, block: { type: 'tool-call', id: CallId('c2'), name: 'side_effect', arguments: '{}' } },
  146. { type: 'usage', usage: { inputTokens: 10, outputTokens: 5 } },
  147. { type: 'finish', reason: { kind: 'tool-calls' } },
  148. ] as Script[number]
  149. const { ctx, parent } = await setup([response])
  150. let sideEffectRan = false
  151. ctx.tools.register(defineContentToolFixture({
  152. name: 'side_effect',
  153. description: 'probe',
  154. parameters: {},
  155. execute(): Promise<ContentBlock[]> {
  156. sideEffectRan = true
  157. return Promise.resolve([{ type: 'text', text: 'ran' }])
  158. },
  159. }))
  160. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  161. const result = await run.result
  162. expect(result.stopReason).toBe('completed')
  163. expect(result.structured).toEqual({ answer: 5 })
  164. // The deny skipped dispatch entirely: the probe body never ran.
  165. expect(sideEffectRan).toBe(false)
  166. await run.dispose()
  167. })
  168. it('a later prepended pre-execute listener cannot resurrect dispatch after capture', async () => {
  169. const response = [
  170. ...toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 5 }).slice(0, -2),
  171. { type: 'block-start', index: 1, blockType: 'tool-call' },
  172. { type: 'block-end', index: 1, block: { type: 'tool-call', id: CallId('c2'), name: 'side_effect', arguments: '{}' } },
  173. { type: 'usage', usage: { inputTokens: 10, outputTokens: 5 } },
  174. { type: 'finish', reason: { kind: 'tool-calls' } },
  175. ] as Script[number]
  176. const { ctx, parent } = await setup([response])
  177. let sideEffectRan = false
  178. ctx.tools.register(defineContentToolFixture({
  179. name: 'side_effect',
  180. description: 'probe',
  181. parameters: {},
  182. execute(): Promise<ContentBlock[]> {
  183. sideEffectRan = true
  184. return Promise.resolve([{ type: 'text', text: 'ran' }])
  185. },
  186. }))
  187. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  188. // Registered after the child and prepended: this listener returns allow
  189. // after every downstream pre-execute decision. The service-owned guard
  190. // runs after the waterfall and can only deny, so the body still cannot run.
  191. ctx.on('tools/pre-execute', async (_exec, next) => {
  192. await next()
  193. return { kind: 'allow' as const }
  194. }, { prepend: true })
  195. const result = await run.result
  196. expect(result.structured).toEqual({ answer: 5 })
  197. expect(sideEffectRan).toBe(false)
  198. const child = ctx.agents.get(run.id)
  199. const sideEffectResult = child?.session.events.find(event =>
  200. event.type === 'tool/result' && event.data.message.source.callId === 'c2')
  201. expect(sideEffectResult?.type === 'tool/result' && sideEffectResult.data.message.content[0].isError).toBe(true)
  202. await run.dispose()
  203. })
  204. it('leaves tool calls that PRECEDE the capture in the same response untouched', async () => {
  205. const response = [
  206. { type: 'block-start', index: 0, blockType: 'tool-call' },
  207. { type: 'block-end', index: 0, block: { type: 'tool-call', id: CallId('c1'), name: 'side_effect', arguments: '{}' } },
  208. ...toolCallResponse('c2', STRUCTURED_OUTPUT_TOOL, { answer: 6 }).map(chunk =>
  209. 'index' in chunk ? { ...chunk, index: 1 } : chunk),
  210. ] as Script[number]
  211. const { ctx, parent } = await setup([response])
  212. let sideEffectRan = false
  213. ctx.tools.register(defineContentToolFixture({
  214. name: 'side_effect',
  215. description: 'probe',
  216. parameters: {},
  217. execute(): Promise<ContentBlock[]> {
  218. sideEffectRan = true
  219. return Promise.resolve([{ type: 'text', text: 'ran' }])
  220. },
  221. }))
  222. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  223. const result = await run.result
  224. // The call ran BEFORE captured was set: the deny gate only guards the
  225. // window after the terminal answer landed.
  226. expect(sideEffectRan).toBe(true)
  227. expect(result.structured).toEqual({ answer: 6 })
  228. await run.dispose()
  229. })
  230. it('an invalid call gets an INVALID_ARGS isError result and the model retries in-turn', async () => {
  231. const { ctx, parent } = await setup([
  232. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 'not-a-number' }),
  233. toolCallResponse('c2', STRUCTURED_OUTPUT_TOOL, { answer: 7 }),
  234. ])
  235. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  236. const result = await run.result
  237. expect(result.structured).toEqual({ answer: 7 })
  238. expect(result.stopReason).toBe('completed')
  239. // The child's log carries the isError tool/result for the invalid call.
  240. const child = ctx.agents.get(run.id)!
  241. const results = child.session.events.filter(e => e.type === 'tool/result')
  242. expect(results.length).toBe(2)
  243. expect(results[0]!.data.message.content[0].isError).toBe(true)
  244. await run.dispose()
  245. })
  246. it('a clean finish without a capture is an immediate error to the parent — deliberately NO re-prompt', async () => {
  247. const { ctx, parent, adapter } = await setup([
  248. textResponse('here is my answer in prose'),
  249. textResponse('MUST NOT BE CONSUMED'),
  250. ])
  251. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  252. const result = await run.result
  253. expect(result.stopReason).toBe('error')
  254. expect(result.structured).toBeUndefined()
  255. // Exactly one model request and one user message: no nudge turn exists.
  256. expect(adapter.requests.length).toBe(1)
  257. const child = ctx.agents.get(run.id)!
  258. expect(child.session.events.filter(e => e.type === 'user/message').length).toBe(1)
  259. await run.dispose()
  260. })
  261. it('an errored child keeps its honest error result (no capture expected)', async () => {
  262. // Script exhaustion on the first call → the child turn errors.
  263. const { ctx, parent, adapter } = await setup([])
  264. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  265. const result = await run.result
  266. expect(result.stopReason).toBe('error')
  267. expect(adapter.requests.length).toBe(1)
  268. await run.dispose()
  269. })
  270. it('a cancel landing after a clean capture-less turn settles aborted, not error', async () => {
  271. const { ctx, parent } = await setup([textResponse('prose, no capture')])
  272. const controller = new AbortController()
  273. const run = await ctx.subagents.start('spawn', structuredRequest(parent, { signal: controller.signal }))
  274. // Cancel synchronously inside the turn's end recording: the cancel
  275. // contract outranks the schema shortfall, so the result maps to aborted.
  276. ctx.on('session/event', (session, event) => {
  277. const child = ctx.agents.get(run.id)
  278. if (session === child?.session && event.type === 'turn/end') controller.abort('cancelled at turn end')
  279. })
  280. const result = await run.result
  281. expect(result.stopReason).toBe('aborted')
  282. await run.dispose()
  283. })
  284. it('rejects a schema outside the subset loud, before any child exists', async () => {
  285. const { ctx, parent } = await setup([])
  286. await expect(ctx.subagents.start('spawn', structuredRequest(parent, {
  287. outputSchema: { type: 'object', oneOf: [] } as unknown as ObjectJsonSchema,
  288. }))).rejects.toThrow(/unsupported JSON schema/)
  289. expect(ctx.agents.get(SessionId('parent'))).toBeDefined()
  290. })
  291. it('a schema carrying non-JSON values fails as JsonSchemaError at the validation boundary', async () => {
  292. const { ctx, parent } = await setup([])
  293. // Semantic assertion runs before provider startup.
  294. await expect(ctx.subagents.start('spawn', structuredRequest(parent, {
  295. outputSchema: { type: 'object', default: () => {} } as unknown as ObjectJsonSchema,
  296. }))).rejects.toThrow(/unsupported JSON schema.*annotation must be lossless JSON data/)
  297. })
  298. it('a post-execute BLOCK on the capture call denies the capture: log and result agree on failure', async () => {
  299. const { ctx, parent, adapter } = await setup([
  300. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 7 }),
  301. textResponse('continues after the blocked capture'),
  302. ])
  303. // A PostToolUse-style hook turns the tool body's provisional success into
  304. // the authoritative final error observed by the commit notification.
  305. ctx.on('tools/post-execute', (exec, _result, next) => {
  306. if (exec.name === STRUCTURED_OUTPUT_TOOL) {
  307. return Promise.resolve({ kind: 'block' as const, feedback: [{ type: 'text' as const, text: 'capture rejected by hook' }] })
  308. }
  309. return next()
  310. })
  311. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  312. const result = await run.result
  313. // No capture was committed: the run reports the schema shortfall...
  314. expect(result.structured).toBeUndefined()
  315. expect(result.stopReason).toBe('error')
  316. // ...the logged tool result is the blocked isError with the feedback...
  317. const child = ctx.agents.get(run.id)!
  318. const results = child.session.events.filter(e => e.type === 'tool/result')
  319. expect(results[0]!.data.message.content[0].isError).toBe(true)
  320. expect(JSON.stringify(results[0]!.data.message.content)).toContain('capture rejected by hook')
  321. // ...and the turn CONTINUED past the blocked call (no captured veto):
  322. // the model got to react to the failure with a second step.
  323. expect(adapter.requests.length).toBe(2)
  324. await run.dispose()
  325. })
  326. it('a post-execute accept-with-replacement still commits the capture', async () => {
  327. const { ctx, parent } = await setup([
  328. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 8 }),
  329. ])
  330. ctx.on('tools/post-execute', (exec, _result, next) => {
  331. if (exec.name === STRUCTURED_OUTPUT_TOOL) {
  332. return Promise.resolve({ kind: 'accept' as const, content: [{ type: 'text' as const, text: 'recorded (rewritten)' }] })
  333. }
  334. return next()
  335. })
  336. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  337. const result = await run.result
  338. expect(result.stopReason).toBe('completed')
  339. expect(result.structured).toEqual({ answer: 8 })
  340. await run.dispose()
  341. })
  342. it('commits only after a later prepended post-execute wrapper returns the authoritative result', async () => {
  343. const { ctx, parent } = await setup([
  344. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 8 }),
  345. textResponse('capture was rejected'),
  346. ])
  347. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  348. // Registered after attachment and prepended, so it wraps every listener
  349. // the child installed. It delegates first, then converts the apparent
  350. // capture success into the pipeline's authoritative failure.
  351. ctx.on('tools/post-execute', async (exec, _result, next) => {
  352. const downstream = await next()
  353. if (exec.name !== STRUCTURED_OUTPUT_TOOL) return downstream
  354. return { kind: 'block' as const, feedback: [{ type: 'text' as const, text: 'rejected after downstream' }] }
  355. }, { prepend: true })
  356. const result = await run.result
  357. expect(result.structured).toBeUndefined()
  358. expect(result.stopReason).toBe('error')
  359. const child = ctx.agents.get(run.id)
  360. const captureResult = child?.session.events.find(event =>
  361. event.type === 'tool/result' && event.data.message.source.callId === 'c1')
  362. expect(captureResult?.type === 'tool/result' && captureResult.data.message.content[0].isError).toBe(true)
  363. await run.dispose()
  364. })
  365. it('appends the structured instruction to the child REQUEST\'s system text (base prompt preserved)', async () => {
  366. const { ctx, parent, adapter } = await setup([toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 1 })])
  367. // A context-wide section stands in for the deployment persona: the
  368. // instruction must APPEND to the other scoped and global sections, not
  369. // replace them (AgentOptions has no prompt field — the instruction is an
  370. // ordinary child-scoped prompt registration).
  371. ctx.systemPrompt.section({ name: 'test:persona', order: 10, text: 'You are a counter.' })
  372. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  373. await run.result
  374. const childRequest = adapter.requests.at(-1)!
  375. expect(childRequest.system).toContain('You are a counter.')
  376. expect(childRequest.system!.endsWith(STRUCTURED_OUTPUT_INSTRUCTION)).toBe(true)
  377. expect(childRequest.system!.indexOf(STRUCTURED_OUTPUT_INSTRUCTION)).toBeGreaterThan(0)
  378. await run.dispose()
  379. })
  380. it('keeps pure Code Mode at one wire tool and exposes structured capture through the SDK only', async () => {
  381. const { ctx, parent, adapter } = await setup([
  382. toolCallResponse('c1', RUN_CODE_NAME, { code: 'return await tools.structured_output({ answer: 12 })', description: 'Capture the structured answer' }),
  383. ], {
  384. toolMode: 'code',
  385. codeRun: async (request) => {
  386. const capture = request.bindings.at(0)?.functions[STRUCTURED_OUTPUT_TOOL]
  387. if (!capture) throw new Error('structured_output binding missing')
  388. await capture({ answer: 12 })
  389. return { logs: [], value: 'captured' }
  390. },
  391. })
  392. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  393. const result = await run.result
  394. expect(result.structured).toEqual({ answer: 12 })
  395. const request = adapter.requests[0]!
  396. expect(toolNames(request)).toEqual([RUN_CODE_NAME])
  397. expect(request.system).toContain('interface ToolArgsMap')
  398. expect(request.system).toContain('interface ToolOutputMap')
  399. expect(request.system).toContain('recorded: true;')
  400. expect(request.system).toContain('Promise<ToolOutputMap[K]>')
  401. expect(request.system).toContain(STRUCTURED_OUTPUT_INSTRUCTION)
  402. await run.dispose()
  403. })
  404. it('discards a nested capture when the enclosing run_code execution fails', async () => {
  405. const { ctx, parent, adapter } = await setup([
  406. toolCallResponse('c1', RUN_CODE_NAME, { code: 'await tools.structured_output({ answer: 12 }); throw new Error("boom")', description: 'Capture then fail the program' }),
  407. textResponse('outer code failed'),
  408. ], {
  409. toolMode: 'code',
  410. codeRun: async (request) => {
  411. const capture = request.bindings.at(0)?.functions[STRUCTURED_OUTPUT_TOOL]
  412. if (!capture) throw new Error('structured_output binding missing')
  413. await capture({ answer: 12 })
  414. return {
  415. logs: [],
  416. error: { kind: 'runtime', message: 'boom after capture' },
  417. } as never
  418. },
  419. })
  420. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  421. const result = await run.result
  422. expect(result.structured).toBeUndefined()
  423. expect(result.stopReason).toBe('error')
  424. expect(adapter.requests).toHaveLength(2)
  425. const child = ctx.agents.get(run.id)!
  426. const outer = child.session.events.find(event =>
  427. event.type === 'tool/result' && event.data.message.source.callId === CallId('c1'))
  428. expect(outer?.type === 'tool/result' && outer.data.message.content[0].isError).toBe(true)
  429. await run.dispose()
  430. })
  431. it('discards a nested capture when post-policy blocks the enclosing run_code result', async () => {
  432. const { ctx, parent, adapter } = await setup([
  433. toolCallResponse('c1', RUN_CODE_NAME, { code: 'return await tools.structured_output({ answer: 12 })', description: 'Capture the structured answer' }),
  434. textResponse('outer code was blocked'),
  435. ], {
  436. toolMode: 'code',
  437. codeRun: async (request) => {
  438. const capture = request.bindings.at(0)?.functions[STRUCTURED_OUTPUT_TOOL]
  439. if (!capture) throw new Error('structured_output binding missing')
  440. await capture({ answer: 12 })
  441. return { logs: [], value: 'captured' }
  442. },
  443. })
  444. ctx.on('tools/post-execute', (exec, _result, next) => exec.name === RUN_CODE_NAME
  445. ? Promise.resolve({ kind: 'block' as const, feedback: [{ type: 'text' as const, text: 'outer blocked' }] })
  446. : next())
  447. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  448. const result = await run.result
  449. expect(result.structured).toBeUndefined()
  450. expect(result.stopReason).toBe('error')
  451. expect(adapter.requests).toHaveLength(2)
  452. await run.dispose()
  453. })
  454. it('the instruction rides ONLY structured requests: appended for the child, absent for a plain agent', async () => {
  455. const { ctx, parent, adapter } = await setup([
  456. textResponse('parent answer'),
  457. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 1 }),
  458. ])
  459. parent.followup(createUserMessage({ content: [{ type: 'text', text: 'hello' }], source: { kind: 'user' } }))
  460. await parent.whenIdle()
  461. expect(adapter.requests[0]!.system ?? '').not.toContain(STRUCTURED_OUTPUT_INSTRUCTION)
  462. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  463. await run.result
  464. // The loop always assembles a base prompt (the harness identity section),
  465. // so the instruction APPENDS — never replaces.
  466. const childSystem = adapter.requests.at(-1)!.system!
  467. expect(childSystem.endsWith(STRUCTURED_OUTPUT_INSTRUCTION)).toBe(true)
  468. expect(childSystem.length).toBeGreaterThan(STRUCTURED_OUTPUT_INSTRUCTION.length)
  469. await run.dispose()
  470. })
  471. describe('scoped registration (each child owns its capture tool)', () => {
  472. it('a plain agent never sees the tool: nothing is registered globally at all', async () => {
  473. const { ctx, parent, adapter } = await setup([textResponse('parent answer')])
  474. parent.followup(createUserMessage({ content: [{ type: 'text', text: 'hello' }], source: { kind: 'user' } }))
  475. await parent.whenIdle()
  476. // Scoped registration: the global view has no capture tool, ever.
  477. expect(ctx.tools.get(STRUCTURED_OUTPUT_TOOL)).toBeUndefined()
  478. expect(toolNames(adapter.requests[0]!)).not.toContain(STRUCTURED_OUTPUT_TOOL)
  479. })
  480. it('a structured child sees structured_output with ITS schema; a plain agent never sees the tool', async () => {
  481. const { ctx, parent, adapter } = await setup([
  482. // Parent turn (a plain agent): must NOT see the tool.
  483. textResponse('parent answer'),
  484. // Child turn: must see it, with the run's schema.
  485. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 42 }),
  486. ])
  487. parent.followup(createUserMessage({ content: [{ type: 'text', text: 'hello' }], source: { kind: 'user' } }))
  488. await parent.whenIdle()
  489. expect(toolNames(adapter.requests[0]!)).not.toContain(STRUCTURED_OUTPUT_TOOL)
  490. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  491. await run.result
  492. const childRequest = adapter.requests[1]!
  493. expect(toolNames(childRequest)).toContain(STRUCTURED_OUTPUT_TOOL)
  494. const entry = childRequest.tools!.find(tool => tool.name === STRUCTURED_OUTPUT_TOOL)!
  495. expect(entry.parameters).toEqual(SCHEMA)
  496. await run.dispose()
  497. })
  498. it('two concurrent structured children each see their OWN schema', async () => {
  499. const otherSchema: ObjectJsonSchema = {
  500. type: 'object',
  501. properties: { verdict: { type: 'string', enum: ['real', 'bogus'] } },
  502. required: ['verdict'],
  503. }
  504. const { ctx, parent, adapter } = await setup([
  505. (options: GenerateOptions) => {
  506. // Answer with whatever schema this child was given — proves each
  507. // request carried the right one regardless of scheduling order.
  508. const entry = options.tools!.find(tool => tool.name === STRUCTURED_OUTPUT_TOOL)!
  509. const args = 'verdict' in (entry.parameters.properties as Record<string, unknown>)
  510. ? { verdict: 'real' }
  511. : { answer: 1 }
  512. return toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, args)
  513. },
  514. (options: GenerateOptions) => {
  515. const entry = options.tools!.find(tool => tool.name === STRUCTURED_OUTPUT_TOOL)!
  516. const args = 'verdict' in (entry.parameters.properties as Record<string, unknown>)
  517. ? { verdict: 'real' }
  518. : { answer: 1 }
  519. return toolCallResponse('c2', STRUCTURED_OUTPUT_TOOL, args)
  520. },
  521. ])
  522. const runA = await ctx.subagents.start('spawn', structuredRequest(parent))
  523. const runB = await ctx.subagents.start('spawn', structuredRequest(parent, { outputSchema: otherSchema }))
  524. const [a, b] = await Promise.all([runA.result, runB.result])
  525. expect(a.structured).toEqual({ answer: 1 })
  526. expect(b.structured).toEqual({ verdict: 'real' })
  527. const schemas = adapter.requests.map(request =>
  528. request.tools!.find(tool => tool.name === STRUCTURED_OUTPUT_TOOL)!.parameters)
  529. expect(schemas).toContainEqual(SCHEMA)
  530. expect(schemas).toContainEqual(otherSchema)
  531. await runA.dispose()
  532. await runB.dispose()
  533. })
  534. it('places the capture tool and instruction in their canonical orders', async () => {
  535. const { ctx, parent, adapter } = await setup([
  536. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 7 }),
  537. ])
  538. // A global tool sorts lexicographically after structured_output, while a
  539. // global section above the 190 band follows the capture instruction.
  540. ctx.tools.register(defineContentToolFixture({
  541. name: 'zz_probe',
  542. description: 'probe',
  543. parameters: {},
  544. execute: () => Promise.resolve([{ type: 'text', text: 'x' }]),
  545. }))
  546. ctx.systemPrompt.section({ name: 'after-band', order: 200, text: 'AFTER-BAND' })
  547. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  548. await run.result
  549. const request = adapter.requests[0]!
  550. const names = toolNames(request)
  551. expect(names.indexOf(STRUCTURED_OUTPUT_TOOL)).toBeGreaterThanOrEqual(0)
  552. expect(names.indexOf(STRUCTURED_OUTPUT_TOOL)).toBeLessThan(names.indexOf('zz_probe'))
  553. const system = request.system ?? ''
  554. const instructionAt = system.indexOf(STRUCTURED_OUTPUT_INSTRUCTION)
  555. expect(instructionAt).toBeGreaterThanOrEqual(0)
  556. expect(system.indexOf('AFTER-BAND')).toBeGreaterThan(instructionAt)
  557. await run.dispose()
  558. })
  559. it('a non-structured agent request keeps tools ABSENT when it had none (no tools: [] materialized)', async () => {
  560. const { parent, adapter } = await setup([textResponse('plain')])
  561. parent.followup(createUserMessage({ content: [{ type: 'text', text: 'q' }], source: { kind: 'user' } }))
  562. await parent.whenIdle()
  563. const request = adapter.requests[0]!
  564. expect(request.tools).toBeUndefined()
  565. await new Promise(resolve => setTimeout(resolve, 0))
  566. })
  567. it('registrations ride the child fiber: disposing the run removes them; a provider reload mid-run cannot', async () => {
  568. const { ctx, parent, disposeProvider } = await setup([
  569. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 4 }),
  570. ])
  571. expect(ctx.tools.get(STRUCTURED_OUTPUT_TOOL)).toBeUndefined()
  572. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  573. // A backend hot-reload mid-run must not unregister the capture tool out
  574. // from under the live child: the registration rides the CHILD's fiber.
  575. disposeProvider()
  576. const result = await run.result
  577. expect(result.structured).toEqual({ answer: 4 })
  578. const child = ctx.agents.get(run.id)!
  579. expect(ctx.tools.get(STRUCTURED_OUTPUT_TOOL, child)).toBeDefined()
  580. await run.dispose()
  581. // Child disposed ⇒ its scoped registrations are gone.
  582. expect(ctx.tools.get(STRUCTURED_OUTPUT_TOOL, child)).toBeUndefined()
  583. })
  584. })
  585. it('a structured_output call from an agent WITHOUT a structured run is UNKNOWN_TOOL (the tool does not exist for it)', async () => {
  586. const { ctx, parent } = await setup([])
  587. const result = await ctx.tools.execute({
  588. signal: testToolSignal,
  589. callId: 'x' as never,
  590. name: STRUCTURED_OUTPUT_TOOL,
  591. arguments: { answer: 1 },
  592. agent: parent,
  593. })
  594. expect(result.isError).toBe(true)
  595. expect(result.error?.info?.code).toBe('UNKNOWN_TOOL')
  596. })
  597. it('a structured_output call with NO calling agent at all is UNKNOWN_TOOL', async () => {
  598. const { ctx } = await setup([])
  599. const result = await ctx.tools.execute({
  600. signal: testToolSignal,
  601. callId: 'x' as never,
  602. name: STRUCTURED_OUTPUT_TOOL,
  603. arguments: { answer: 1 },
  604. })
  605. expect(result.isError).toBe(true)
  606. expect(result.error?.info?.code).toBe('UNKNOWN_TOOL')
  607. })
  608. it('a failed execution stage is discarded and never promoted by a later call', async () => {
  609. const { ctx, parent } = await setup([
  610. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 1 }),
  611. ])
  612. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  613. // A prepended post-execute listener blocks the first capture without
  614. // delegating. The final-result notification discards that execution's
  615. // stage when it observes the error.
  616. let blocks = 1
  617. ctx.on('tools/post-execute', (exec, _result, next) => {
  618. if (exec.name === STRUCTURED_OUTPUT_TOOL && blocks > 0) {
  619. blocks -= 1
  620. return Promise.resolve({ kind: 'block' as const, feedback: [{ type: 'text' as const, text: 'rejected' }] })
  621. }
  622. return next()
  623. }, { prepend: true })
  624. const result = await run.result
  625. const child = ctx.agents.get(run.id)!
  626. // The blocked capture must NOT surface as structured success…
  627. expect(result.stopReason).toBe('error')
  628. expect(result.structured).toBeUndefined()
  629. // …and a LATER invalid call (its own body staged nothing) must not
  630. // resurrect c1's discarded value: drive the pipeline directly.
  631. const invalid = await ctx.tools.execute({
  632. signal: testToolSignal,
  633. callId: 'c2' as never,
  634. name: STRUCTURED_OUTPUT_TOOL,
  635. arguments: { answer: 'not-a-number' },
  636. agent: child,
  637. })
  638. expect(invalid.isError).toBe(true)
  639. // A fresh valid call still captures ITS OWN value.
  640. const valid = await ctx.tools.execute({
  641. signal: testToolSignal,
  642. callId: 'c3' as never,
  643. name: STRUCTURED_OUTPUT_TOOL,
  644. arguments: { answer: 9 },
  645. agent: child,
  646. })
  647. expect(valid.isError).toBeFalsy()
  648. await run.dispose()
  649. })
  650. it('reusing a failed execution\'s call id never promotes its discarded stage', async () => {
  651. const { ctx, parent } = await setup([
  652. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 1 }),
  653. ])
  654. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  655. // Block the first capture after its body stages a value. Its final error
  656. // discards that execution's stage.
  657. let blocks = 1
  658. ctx.on('tools/post-execute', (exec, _result, next) => {
  659. if (exec.name === STRUCTURED_OUTPUT_TOOL && blocks > 0) {
  660. blocks -= 1
  661. return Promise.resolve({ kind: 'block' as const, feedback: [{ type: 'text' as const, text: 'rejected' }] })
  662. }
  663. return next()
  664. }, { prepend: true })
  665. await run.result
  666. const child = ctx.agents.get(run.id)!
  667. // A SECOND capture call with the SAME call id whose body never stages
  668. // (invalid args throw before the stage): the discarded value must not ride
  669. // its acceptance.
  670. const reused = await ctx.tools.execute({
  671. signal: testToolSignal,
  672. callId: 'c1' as never,
  673. name: STRUCTURED_OUTPUT_TOOL,
  674. arguments: { answer: 'not-a-number' },
  675. agent: child,
  676. })
  677. expect(reused.isError).toBe(true)
  678. // Nothing was ever committed: a fresh valid call is still required.
  679. const valid = await ctx.tools.execute({
  680. signal: testToolSignal,
  681. callId: 'c1' as never,
  682. name: STRUCTURED_OUTPUT_TOOL,
  683. arguments: { answer: 5 },
  684. agent: child,
  685. })
  686. expect(valid.isError).toBeFalsy()
  687. await run.dispose()
  688. })
  689. it('a pre-execute deny with call-id reuse cannot promote another execution\'s stage', async () => {
  690. const { ctx, parent } = await setup([
  691. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 1 }),
  692. ])
  693. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  694. // Discard the first capture's stage via a final post-execute block.
  695. let blocks = 1
  696. ctx.on('tools/post-execute', (exec, _result, next) => {
  697. if (exec.name === STRUCTURED_OUTPUT_TOOL && blocks > 0) {
  698. blocks -= 1
  699. return Promise.resolve({ kind: 'block' as const, feedback: [{ type: 'text' as const, text: 'rejected' }] })
  700. }
  701. return next()
  702. }, { prepend: true })
  703. await run.result
  704. const child = ctx.agents.get(run.id)!
  705. // A prepended pre-execute deny skips the body, while the denied call still
  706. // reaches the final notification with the same adapter-minted call id.
  707. const offDeny = ctx.on('tools/pre-execute', (exec) => {
  708. if (exec.name === STRUCTURED_OUTPUT_TOOL) {
  709. return Promise.resolve({ kind: 'deny' as const, reason: 'outer veto' })
  710. }
  711. return undefined as never
  712. }, { prepend: true })
  713. const denied = await ctx.tools.execute({
  714. signal: testToolSignal,
  715. callId: 'c1' as never,
  716. name: STRUCTURED_OUTPUT_TOOL,
  717. arguments: { answer: 2 },
  718. agent: child,
  719. })
  720. expect(denied.isError).toBe(true)
  721. offDeny()
  722. // The discarded value was never promoted: a fresh valid call is required
  723. // (and succeeds, proving the runtime is not wedged).
  724. const valid = await ctx.tools.execute({
  725. signal: testToolSignal,
  726. callId: 'c1' as never,
  727. name: STRUCTURED_OUTPUT_TOOL,
  728. arguments: { answer: 5 },
  729. agent: child,
  730. })
  731. expect(valid.isError).toBeFalsy()
  732. await run.dispose()
  733. })
  734. })