structured.spec.ts 37 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805
  1. import { describe, expect, it } from 'vitest'
  2. import { Context } from 'cordis'
  3. import { CallId, type ContentBlock, type GenerateOptions } from '@deepseek-ai/dsh-llm'
  4. import type { ContinuationDecision } from '@deepseek-ai/dsh-agent'
  5. import { SessionId } from '@deepseek-ai/dsh-session'
  6. import AgentLoop from '@deepseek-ai/dsh-agent-loop'
  7. import { mountAgentLoopTestDependencies } from '@deepseek-ai/dsh-agent-loop-testkit'
  8. import InvariantService from '@deepseek-ai/dsh-invariants'
  9. import * as SessionInvariant from '@deepseek-ai/dsh-session/invariant'
  10. import * as AgentInvariant from '@deepseek-ai/dsh-agent/invariant'
  11. import * as AgentLoopInvariant from '@deepseek-ai/dsh-agent-loop/invariant'
  12. import SubagentService, { type SubagentStartRequest } from '@deepseek-ai/dsh-subagent'
  13. import type { Config as ToolConfig, StructuredOutputSchema } from '@deepseek-ai/dsh-tools'
  14. import { RUN_CODE_NAME } from '@deepseek-ai/dsh-tools'
  15. import { MockAdapter, textResponse, toolCallResponse } from '../../../core/agent-loop/tests/mock-adapter.ts'
  16. import { startInProcessRun } from '../src/index.ts'
  17. import {
  18. STRUCTURED_OUTPUT_INSTRUCTION,
  19. STRUCTURED_OUTPUT_TOOL,
  20. } from '../src/structured.ts'
  21. const testToolSignal = new AbortController().signal
  22. type Script = ConstructorParameters<typeof MockAdapter>[0]
  23. async function mountInvariants(ctx: Context): Promise<void> {
  24. await ctx.plugin(InvariantService)
  25. await ctx.plugin(SessionInvariant)
  26. await ctx.plugin(AgentInvariant)
  27. await ctx.plugin(AgentLoopInvariant)
  28. }
  29. interface CodeRunRequestLike {
  30. bindings: { global: string; functions: Record<string, (args: unknown) => Promise<unknown>> }[]
  31. }
  32. interface SetupOptions {
  33. toolMode?: ToolConfig['mode']
  34. codeRun?: (request: CodeRunRequestLike) => Promise<{ logs: never[]; value?: unknown }>
  35. }
  36. const SCHEMA: StructuredOutputSchema = {
  37. type: 'object',
  38. properties: { answer: { type: 'number' }, note: { type: 'string' } },
  39. required: ['answer'],
  40. }
  41. /**
  42. * Real loop, scripted model, and inline fresh-conversation provider over the shared driver. Loading
  43. * spawn/fork here would create a dev-dependency cycle; their specs cover plugin integration while
  44. * this fixture isolates driver behavior and scripts the child's `structured_output` calls.
  45. */
  46. async function setup(script: Script, options: SetupOptions = {}) {
  47. const ctx = new Context()
  48. const adapter = new MockAdapter(script)
  49. await mountAgentLoopTestDependencies(ctx, {
  50. tools: { mode: options.toolMode ?? 'native' },
  51. })
  52. if (options.toolMode === 'code' || options.toolMode === 'both') {
  53. ctx.provide('codeRuntime', {
  54. language: 'typescript',
  55. isolation: 'test',
  56. run: options.codeRun ?? (() => Promise.resolve({ logs: [] })),
  57. } as never)
  58. }
  59. await mountInvariants(ctx)
  60. await ctx.plugin(AgentLoop, { agents: [] })
  61. await ctx.plugin(SubagentService)
  62. const disposeProvider = ctx.subagents.registerProvider({
  63. name: 'spawn',
  64. capabilities: { outputSchema: true, depthLimit: true, toolFilter: false, persona: false },
  65. inheritsParentContext: false,
  66. start: (request: SubagentStartRequest) => startInProcessRun(request, {}),
  67. })
  68. ctx.llm.registerAdapter(['mock'], adapter)
  69. const parent = ctx.agentLoop.create(SessionId('parent'), { provider: 'mock', model: 'mock' })
  70. return { ctx, parent, adapter, disposeProvider }
  71. }
  72. function structuredRequest(parent: SubagentStartRequest['parent'], extra?: Partial<SubagentStartRequest>): SubagentStartRequest {
  73. return {
  74. prompt: [{ type: 'text', text: 'produce the answer' }],
  75. parent,
  76. signal: new AbortController().signal,
  77. outputSchema: SCHEMA,
  78. ...extra,
  79. }
  80. }
  81. /** The tool names of one recorded model request. */
  82. function toolNames(request: GenerateOptions): string[] {
  83. return (request.tools ?? []).map(tool => tool.name)
  84. }
  85. describe('in-process structured output', () => {
  86. it('captures a valid structured_output call and surfaces result.structured', async () => {
  87. const { ctx, parent } = await setup([
  88. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 42, note: 'done' }),
  89. ])
  90. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  91. const result = await run.result
  92. expect(result.stopReason).toBe('completed')
  93. expect(result.structured).toEqual({ answer: 42, note: 'done' })
  94. await run.dispose()
  95. })
  96. it('stops the turn after a successful capture — no extra model step is spent', async () => {
  97. const { ctx, parent, adapter } = await setup([
  98. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 1 }),
  99. textResponse('MUST NOT BE CONSUMED'),
  100. ])
  101. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  102. await run.result
  103. // Default continuation would run a second step after the tool call; the
  104. // structured runtime's turn-continuation veto stops the turn instead.
  105. expect(adapter.requests.length).toBe(1)
  106. await run.dispose()
  107. })
  108. it('denies tool calls that FOLLOW the capture in the same response — terminal means terminal', async () => {
  109. // One model response carrying structured_output FIRST and a side-effecting
  110. // call after it: the continuation veto only fires at step end, so without
  111. // the pre-execute deny the trailing call would still run after the final
  112. // answer was accepted.
  113. const response = [
  114. ...toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 5 }).slice(0, -2),
  115. { type: 'block-start', index: 1, blockType: 'tool-call' },
  116. { type: 'block-end', index: 1, block: { type: 'tool-call', id: CallId('c2'), name: 'side_effect', arguments: '{}' } },
  117. { type: 'usage', usage: { inputTokens: 10, outputTokens: 5 } },
  118. { type: 'finish', reason: { kind: 'tool-calls' } },
  119. ] as Script[number]
  120. const { ctx, parent } = await setup([response])
  121. let sideEffectRan = false
  122. ctx.tools.register({
  123. name: 'side_effect',
  124. description: 'probe',
  125. parameters: { type: 'object', properties: {} },
  126. execute(): Promise<ContentBlock[]> {
  127. sideEffectRan = true
  128. return Promise.resolve([{ type: 'text', text: 'ran' }])
  129. },
  130. })
  131. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  132. const result = await run.result
  133. expect(result.stopReason).toBe('completed')
  134. expect(result.structured).toEqual({ answer: 5 })
  135. // The deny skipped dispatch entirely: the probe body never ran.
  136. expect(sideEffectRan).toBe(false)
  137. await run.dispose()
  138. })
  139. it('a later prepended pre-execute listener cannot resurrect dispatch after capture', async () => {
  140. const response = [
  141. ...toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 5 }).slice(0, -2),
  142. { type: 'block-start', index: 1, blockType: 'tool-call' },
  143. { type: 'block-end', index: 1, block: { type: 'tool-call', id: CallId('c2'), name: 'side_effect', arguments: '{}' } },
  144. { type: 'usage', usage: { inputTokens: 10, outputTokens: 5 } },
  145. { type: 'finish', reason: { kind: 'tool-calls' } },
  146. ] as Script[number]
  147. const { ctx, parent } = await setup([response])
  148. let sideEffectRan = false
  149. ctx.tools.register({
  150. name: 'side_effect',
  151. description: 'probe',
  152. parameters: { type: 'object', properties: {} },
  153. execute(): Promise<ContentBlock[]> {
  154. sideEffectRan = true
  155. return Promise.resolve([{ type: 'text', text: 'ran' }])
  156. },
  157. })
  158. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  159. // Registered after the child and prepended: this listener returns allow
  160. // after every downstream pre-execute decision. The service-owned guard
  161. // runs after the waterfall and can only deny, so the body still cannot run.
  162. ctx.on('tools/pre-execute', async (_exec, next) => {
  163. await next()
  164. return { kind: 'allow' as const }
  165. }, { prepend: true })
  166. const result = await run.result
  167. expect(result.structured).toEqual({ answer: 5 })
  168. expect(sideEffectRan).toBe(false)
  169. const child = ctx.agents.get(run.id)
  170. const sideEffectResult = child?.session.events.find(event =>
  171. event.type === 'tool/result' && event.data.callId === 'c2')
  172. expect(sideEffectResult?.type === 'tool/result' && sideEffectResult.data.isError).toBe(true)
  173. await run.dispose()
  174. })
  175. it('leaves tool calls that PRECEDE the capture in the same response untouched', async () => {
  176. const response = [
  177. { type: 'block-start', index: 0, blockType: 'tool-call' },
  178. { type: 'block-end', index: 0, block: { type: 'tool-call', id: CallId('c1'), name: 'side_effect', arguments: '{}' } },
  179. ...toolCallResponse('c2', STRUCTURED_OUTPUT_TOOL, { answer: 6 }).map(chunk =>
  180. 'index' in chunk ? { ...chunk, index: 1 } : chunk),
  181. ] as Script[number]
  182. const { ctx, parent } = await setup([response])
  183. let sideEffectRan = false
  184. ctx.tools.register({
  185. name: 'side_effect',
  186. description: 'probe',
  187. parameters: { type: 'object', properties: {} },
  188. execute(): Promise<ContentBlock[]> {
  189. sideEffectRan = true
  190. return Promise.resolve([{ type: 'text', text: 'ran' }])
  191. },
  192. })
  193. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  194. const result = await run.result
  195. // The call ran BEFORE captured was set: the deny gate only guards the
  196. // window after the terminal answer landed.
  197. expect(sideEffectRan).toBe(true)
  198. expect(result.structured).toEqual({ answer: 6 })
  199. await run.dispose()
  200. })
  201. it('a later-prepended continuation wrapper cannot resurrect a captured turn', async () => {
  202. const { ctx, parent, adapter } = await setup([
  203. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 7 }),
  204. textResponse('MUST NOT BE CONSUMED'),
  205. ])
  206. ctx.on('agent/turn-continuation', () => Promise.resolve<ContinuationDecision>({ action: 'stop' }))
  207. let wrapperInstalled = false
  208. // Register before ready-only start: structured output is attached before session-start and the
  209. // loop. The wrapper waits for a downstream stop, rewrites it to continue, and must still lose
  210. // to the later terminal checkpoint.
  211. ctx.on('agent/session-start', (child) => {
  212. if (child === parent) return
  213. wrapperInstalled = true
  214. child.ctx.on('agent/turn-continuation', async (_subject, _turn, _decision, _signal, next): Promise<ContinuationDecision> => {
  215. const downstream = await next()
  216. expect(downstream).toEqual({ action: 'stop' })
  217. return { action: 'continue' }
  218. }, { prepend: true })
  219. })
  220. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  221. const result = await run.result
  222. expect(wrapperInstalled).toBe(true)
  223. expect(result.structured).toEqual({ answer: 7 })
  224. expect(result.stopReason).toBe('completed')
  225. expect(adapter.requests).toHaveLength(1)
  226. await run.dispose()
  227. })
  228. it('a continuation wrapper cannot carry steering past a captured terminal stop', async () => {
  229. const { ctx, parent, adapter } = await setup([
  230. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 9 }),
  231. textResponse('MUST NOT BE CONSUMED'),
  232. ])
  233. // A downstream policy stops, then a later wrapper delegates and queues steering that ordinary
  234. // folding would turn into continue. The terminal checkpoint must discard that steering.
  235. ctx.on('agent/turn-continuation', () => Promise.resolve<ContinuationDecision>({ action: 'stop' }))
  236. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  237. ctx.on('agent/session-start', (child) => {
  238. if (child.id !== run.id) return
  239. child.ctx.on('agent/turn-continuation', async (subject, _turn, _decision, _signal, next): Promise<ContinuationDecision> => {
  240. const downstream = await next()
  241. expect(downstream).toEqual({ action: 'stop' })
  242. subject.steer([{ type: 'text', text: 'late steering after downstream stop' }])
  243. return downstream
  244. }, { prepend: true })
  245. })
  246. const result = await run.result
  247. const child = ctx.agents.get(run.id)
  248. expect(result.structured).toEqual({ answer: 9 })
  249. expect(adapter.requests).toHaveLength(1)
  250. expect(child?.session.events.filter(event => event.type === 'turn/start')).toHaveLength(1)
  251. expect(child?.session.events.filter(event => event.type === 'steering/message')).toHaveLength(0)
  252. await run.dispose()
  253. })
  254. it('an invalid call gets an INVALID_ARGS isError result and the model retries in-turn', async () => {
  255. const { ctx, parent } = await setup([
  256. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 'not-a-number' }),
  257. toolCallResponse('c2', STRUCTURED_OUTPUT_TOOL, { answer: 7 }),
  258. ])
  259. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  260. const result = await run.result
  261. expect(result.structured).toEqual({ answer: 7 })
  262. expect(result.stopReason).toBe('completed')
  263. // The child's log carries the isError tool/result for the invalid call.
  264. const child = ctx.agents.get(run.id)!
  265. const results = child.session.events.filter(e => e.type === 'tool/result')
  266. expect(results.length).toBe(2)
  267. expect((results[0]!.data as { isError?: boolean }).isError).toBe(true)
  268. await run.dispose()
  269. })
  270. it('a clean finish without a capture is an immediate error to the parent — deliberately NO re-prompt', async () => {
  271. const { ctx, parent, adapter } = await setup([
  272. textResponse('here is my answer in prose'),
  273. textResponse('MUST NOT BE CONSUMED'),
  274. ])
  275. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  276. const result = await run.result
  277. expect(result.stopReason).toBe('error')
  278. expect(result.structured).toBeUndefined()
  279. // Exactly one model request and one user message: no nudge turn exists.
  280. expect(adapter.requests.length).toBe(1)
  281. const child = ctx.agents.get(run.id)!
  282. expect(child.session.events.filter(e => e.type === 'user/message').length).toBe(1)
  283. await run.dispose()
  284. })
  285. it('an errored child keeps its honest error result (no capture expected)', async () => {
  286. // Script exhaustion on the first call → the child turn errors.
  287. const { ctx, parent, adapter } = await setup([])
  288. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  289. const result = await run.result
  290. expect(result.stopReason).toBe('error')
  291. expect(adapter.requests.length).toBe(1)
  292. await run.dispose()
  293. })
  294. it('a cancel landing after a clean capture-less turn settles aborted, not error', async () => {
  295. const { ctx, parent } = await setup([textResponse('prose, no capture')])
  296. const controller = new AbortController()
  297. const run = await ctx.subagents.start('spawn', structuredRequest(parent, { signal: controller.signal }))
  298. // Cancel synchronously inside the turn's end recording: the cancel
  299. // contract outranks the schema shortfall, so the result maps to aborted.
  300. ctx.on('session/event', (session, event) => {
  301. const child = ctx.agents.get(run.id)
  302. if (session === child?.session && event.type === 'turn/end') controller.abort('cancelled at turn end')
  303. })
  304. const result = await run.result
  305. expect(result.stopReason).toBe('aborted')
  306. await run.dispose()
  307. })
  308. it('rejects a schema outside the subset loud, before any child exists', async () => {
  309. const { ctx, parent } = await setup([])
  310. await expect(ctx.subagents.start('spawn', structuredRequest(parent, {
  311. outputSchema: { type: 'object', oneOf: [] } as unknown as StructuredOutputSchema,
  312. }))).rejects.toThrow(/unsupported output schema/)
  313. expect(ctx.agents.get(SessionId('parent'))).toBeDefined()
  314. })
  315. it('a schema carrying non-JSON values fails as OutputSchemaError at the validation boundary', async () => {
  316. const { ctx, parent } = await setup([])
  317. // Semantic assertion runs before provider startup.
  318. await expect(ctx.subagents.start('spawn', structuredRequest(parent, {
  319. outputSchema: { type: 'object', default: () => {} } as unknown as StructuredOutputSchema,
  320. }))).rejects.toThrow(/unsupported output schema.*annotation must be JSON data/)
  321. })
  322. it('a post-execute BLOCK on the capture call denies the capture: log and result agree on failure', async () => {
  323. const { ctx, parent, adapter } = await setup([
  324. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 7 }),
  325. textResponse('continues after the blocked capture'),
  326. ])
  327. // A PostToolUse-style hook turns the tool body's provisional success into
  328. // the authoritative final error observed by the commit notification.
  329. ctx.on('tools/post-execute', (exec, _result, next) => {
  330. if (exec.name === STRUCTURED_OUTPUT_TOOL) {
  331. return Promise.resolve({ kind: 'block' as const, feedback: [{ type: 'text' as const, text: 'capture rejected by hook' }] })
  332. }
  333. return next()
  334. })
  335. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  336. const result = await run.result
  337. // No capture was committed: the run reports the schema shortfall...
  338. expect(result.structured).toBeUndefined()
  339. expect(result.stopReason).toBe('error')
  340. // ...the logged tool result is the blocked isError with the feedback...
  341. const child = ctx.agents.get(run.id)!
  342. const results = child.session.events.filter(e => e.type === 'tool/result')
  343. expect((results[0]!.data as { isError?: boolean }).isError).toBe(true)
  344. expect(JSON.stringify((results[0]!.data as { content: unknown }).content)).toContain('capture rejected by hook')
  345. // ...and the turn CONTINUED past the blocked call (no captured veto):
  346. // the model got to react to the failure with a second step.
  347. expect(adapter.requests.length).toBe(2)
  348. await run.dispose()
  349. })
  350. it('a post-execute accept-with-replacement still commits the capture', async () => {
  351. const { ctx, parent } = await setup([
  352. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 8 }),
  353. ])
  354. ctx.on('tools/post-execute', (exec, _result, next) => {
  355. if (exec.name === STRUCTURED_OUTPUT_TOOL) {
  356. return Promise.resolve({ kind: 'accept' as const, content: [{ type: 'text' as const, text: 'recorded (rewritten)' }] })
  357. }
  358. return next()
  359. })
  360. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  361. const result = await run.result
  362. expect(result.stopReason).toBe('completed')
  363. expect(result.structured).toEqual({ answer: 8 })
  364. await run.dispose()
  365. })
  366. it('commits only after a later prepended post-execute wrapper returns the authoritative result', async () => {
  367. const { ctx, parent } = await setup([
  368. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 8 }),
  369. textResponse('capture was rejected'),
  370. ])
  371. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  372. // Registered after attachment and prepended, so it wraps every listener
  373. // the child installed. It delegates first, then converts the apparent
  374. // capture success into the pipeline's authoritative failure.
  375. ctx.on('tools/post-execute', async (exec, _result, next) => {
  376. const downstream = await next()
  377. if (exec.name !== STRUCTURED_OUTPUT_TOOL) return downstream
  378. return { kind: 'block' as const, feedback: [{ type: 'text' as const, text: 'rejected after downstream' }] }
  379. }, { prepend: true })
  380. const result = await run.result
  381. expect(result.structured).toBeUndefined()
  382. expect(result.stopReason).toBe('error')
  383. const child = ctx.agents.get(run.id)
  384. const captureResult = child?.session.events.find(event =>
  385. event.type === 'tool/result' && event.data.callId === 'c1')
  386. expect(captureResult?.type === 'tool/result' && captureResult.data.isError).toBe(true)
  387. await run.dispose()
  388. })
  389. it('appends the structured instruction to the child REQUEST\'s system text (base prompt preserved)', async () => {
  390. const { ctx, parent, adapter } = await setup([toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 1 })])
  391. // A context-wide section stands in for the deployment persona: the
  392. // instruction must APPEND to the other scoped and global sections, not
  393. // replace them (AgentOptions has no prompt field — the instruction is an
  394. // ordinary child-scoped prompt registration).
  395. ctx.systemPrompt.section({ name: 'test:persona', order: 10, text: 'You are a counter.' })
  396. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  397. await run.result
  398. const childRequest = adapter.requests.at(-1)!
  399. expect(childRequest.system).toContain('You are a counter.')
  400. expect(childRequest.system!.endsWith(STRUCTURED_OUTPUT_INSTRUCTION)).toBe(true)
  401. expect(childRequest.system!.indexOf(STRUCTURED_OUTPUT_INSTRUCTION)).toBeGreaterThan(0)
  402. await run.dispose()
  403. })
  404. it('keeps pure Code Mode at one wire tool and exposes structured capture through the SDK only', async () => {
  405. const { ctx, parent, adapter } = await setup([
  406. toolCallResponse('c1', RUN_CODE_NAME, { code: 'return await tools.structured_output({ answer: 12 })' }),
  407. ], {
  408. toolMode: 'code',
  409. codeRun: async (request) => {
  410. const capture = request.bindings.at(0)?.functions[STRUCTURED_OUTPUT_TOOL]
  411. if (!capture) throw new Error('structured_output binding missing')
  412. await capture({ answer: 12 })
  413. return { logs: [], value: 'captured' }
  414. },
  415. })
  416. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  417. const result = await run.result
  418. expect(result.structured).toEqual({ answer: 12 })
  419. const request = adapter.requests[0]!
  420. expect(toolNames(request)).toEqual([RUN_CODE_NAME])
  421. expect(request.system).toContain('declare const tools:')
  422. expect(request.system).toContain('structured_output(args:')
  423. expect(request.system).toContain(STRUCTURED_OUTPUT_INSTRUCTION)
  424. await run.dispose()
  425. })
  426. it('discards a nested capture when the enclosing run_code execution fails', async () => {
  427. const { ctx, parent, adapter } = await setup([
  428. toolCallResponse('c1', RUN_CODE_NAME, { code: 'await tools.structured_output({ answer: 12 }); throw new Error("boom")' }),
  429. textResponse('outer code failed'),
  430. ], {
  431. toolMode: 'code',
  432. codeRun: async (request) => {
  433. const capture = request.bindings.at(0)?.functions[STRUCTURED_OUTPUT_TOOL]
  434. if (!capture) throw new Error('structured_output binding missing')
  435. await capture({ answer: 12 })
  436. return {
  437. logs: [],
  438. error: { kind: 'runtime', message: 'boom after capture' },
  439. } as never
  440. },
  441. })
  442. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  443. const result = await run.result
  444. expect(result.structured).toBeUndefined()
  445. expect(result.stopReason).toBe('error')
  446. expect(adapter.requests).toHaveLength(2)
  447. const child = ctx.agents.get(run.id)!
  448. const outer = child.session.events.find(event =>
  449. event.type === 'tool/result' && event.data.callId === CallId('c1'))
  450. expect(outer?.type === 'tool/result' && outer.data.isError).toBe(true)
  451. await run.dispose()
  452. })
  453. it('discards a nested capture when post-policy blocks the enclosing run_code result', async () => {
  454. const { ctx, parent, adapter } = await setup([
  455. toolCallResponse('c1', RUN_CODE_NAME, { code: 'return await tools.structured_output({ answer: 12 })' }),
  456. textResponse('outer code was blocked'),
  457. ], {
  458. toolMode: 'code',
  459. codeRun: async (request) => {
  460. const capture = request.bindings.at(0)?.functions[STRUCTURED_OUTPUT_TOOL]
  461. if (!capture) throw new Error('structured_output binding missing')
  462. await capture({ answer: 12 })
  463. return { logs: [], value: 'captured' }
  464. },
  465. })
  466. ctx.on('tools/post-execute', (exec, _result, next) => exec.name === RUN_CODE_NAME
  467. ? Promise.resolve({ kind: 'block' as const, feedback: [{ type: 'text' as const, text: 'outer blocked' }] })
  468. : next())
  469. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  470. const result = await run.result
  471. expect(result.structured).toBeUndefined()
  472. expect(result.stopReason).toBe('error')
  473. expect(adapter.requests).toHaveLength(2)
  474. await run.dispose()
  475. })
  476. it('the instruction rides ONLY structured requests: appended for the child, absent for a plain agent', async () => {
  477. const { ctx, parent, adapter } = await setup([
  478. textResponse('parent answer'),
  479. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 1 }),
  480. ])
  481. parent.send([{ type: 'text', text: 'hello' }])
  482. await parent.whenIdle()
  483. expect(adapter.requests[0]!.system ?? '').not.toContain(STRUCTURED_OUTPUT_INSTRUCTION)
  484. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  485. await run.result
  486. // The loop always assembles a base prompt (the harness identity section),
  487. // so the instruction APPENDS — never replaces.
  488. const childSystem = adapter.requests.at(-1)!.system!
  489. expect(childSystem.endsWith(STRUCTURED_OUTPUT_INSTRUCTION)).toBe(true)
  490. expect(childSystem.length).toBeGreaterThan(STRUCTURED_OUTPUT_INSTRUCTION.length)
  491. await run.dispose()
  492. })
  493. describe('scoped registration (each child owns its capture tool)', () => {
  494. it('a plain agent never sees the tool: nothing is registered globally at all', async () => {
  495. const { ctx, parent, adapter } = await setup([textResponse('parent answer')])
  496. parent.send([{ type: 'text', text: 'hello' }])
  497. await parent.whenIdle()
  498. // Scoped registration: the global view has no capture tool, ever.
  499. expect(ctx.tools.get(STRUCTURED_OUTPUT_TOOL)).toBeUndefined()
  500. expect(toolNames(adapter.requests[0]!)).not.toContain(STRUCTURED_OUTPUT_TOOL)
  501. })
  502. it('a structured child sees structured_output with ITS schema; a plain agent never sees the tool', async () => {
  503. const { ctx, parent, adapter } = await setup([
  504. // Parent turn (a plain agent): must NOT see the tool.
  505. textResponse('parent answer'),
  506. // Child turn: must see it, with the run's schema.
  507. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 42 }),
  508. ])
  509. parent.send([{ type: 'text', text: 'hello' }])
  510. await parent.whenIdle()
  511. expect(toolNames(adapter.requests[0]!)).not.toContain(STRUCTURED_OUTPUT_TOOL)
  512. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  513. await run.result
  514. const childRequest = adapter.requests[1]!
  515. expect(toolNames(childRequest)).toContain(STRUCTURED_OUTPUT_TOOL)
  516. const entry = childRequest.tools!.find(tool => tool.name === STRUCTURED_OUTPUT_TOOL)!
  517. expect(entry.parameters).toEqual(SCHEMA)
  518. await run.dispose()
  519. })
  520. it('two concurrent structured children each see their OWN schema', async () => {
  521. const otherSchema: StructuredOutputSchema = {
  522. type: 'object',
  523. properties: { verdict: { type: 'string', enum: ['real', 'bogus'] } },
  524. required: ['verdict'],
  525. }
  526. const { ctx, parent, adapter } = await setup([
  527. (options: GenerateOptions) => {
  528. // Answer with whatever schema this child was given — proves each
  529. // request carried the right one regardless of scheduling order.
  530. const entry = options.tools!.find(tool => tool.name === STRUCTURED_OUTPUT_TOOL)!
  531. const args = 'verdict' in (entry.parameters.properties as Record<string, unknown>)
  532. ? { verdict: 'real' }
  533. : { answer: 1 }
  534. return toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, args)
  535. },
  536. (options: GenerateOptions) => {
  537. const entry = options.tools!.find(tool => tool.name === STRUCTURED_OUTPUT_TOOL)!
  538. const args = 'verdict' in (entry.parameters.properties as Record<string, unknown>)
  539. ? { verdict: 'real' }
  540. : { answer: 1 }
  541. return toolCallResponse('c2', STRUCTURED_OUTPUT_TOOL, args)
  542. },
  543. ])
  544. const runA = await ctx.subagents.start('spawn', structuredRequest(parent))
  545. const runB = await ctx.subagents.start('spawn', structuredRequest(parent, { outputSchema: otherSchema }))
  546. const [a, b] = await Promise.all([runA.result, runB.result])
  547. expect(a.structured).toEqual({ answer: 1 })
  548. expect(b.structured).toEqual({ verdict: 'real' })
  549. const schemas = adapter.requests.map(request =>
  550. request.tools!.find(tool => tool.name === STRUCTURED_OUTPUT_TOOL)!.parameters)
  551. expect(schemas).toContainEqual(SCHEMA)
  552. expect(schemas).toContainEqual(otherSchema)
  553. await runA.dispose()
  554. await runB.dispose()
  555. })
  556. it('places the capture tool and instruction in their canonical orders', async () => {
  557. const { ctx, parent, adapter } = await setup([
  558. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 7 }),
  559. ])
  560. // A global tool sorts lexicographically after structured_output, while a
  561. // global section above the 190 band follows the capture instruction.
  562. ctx.tools.register({
  563. name: 'zz_probe',
  564. description: 'probe',
  565. parameters: { type: 'object', properties: {} },
  566. execute: () => Promise.resolve([{ type: 'text', text: 'x' }]),
  567. })
  568. ctx.systemPrompt.section({ name: 'after-band', order: 200, text: 'AFTER-BAND' })
  569. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  570. await run.result
  571. const request = adapter.requests[0]!
  572. const names = toolNames(request)
  573. expect(names.indexOf(STRUCTURED_OUTPUT_TOOL)).toBeGreaterThanOrEqual(0)
  574. expect(names.indexOf(STRUCTURED_OUTPUT_TOOL)).toBeLessThan(names.indexOf('zz_probe'))
  575. const system = request.system ?? ''
  576. const instructionAt = system.indexOf(STRUCTURED_OUTPUT_INSTRUCTION)
  577. expect(instructionAt).toBeGreaterThanOrEqual(0)
  578. expect(system.indexOf('AFTER-BAND')).toBeGreaterThan(instructionAt)
  579. await run.dispose()
  580. })
  581. it('a non-structured agent request keeps tools ABSENT when it had none (no tools: [] materialized)', async () => {
  582. const { parent, adapter } = await setup([textResponse('plain')])
  583. parent.send([{ type: 'text', text: 'q' }])
  584. await parent.whenIdle()
  585. const request = adapter.requests[0]!
  586. expect(request.tools).toBeUndefined()
  587. await new Promise(resolve => setTimeout(resolve, 0))
  588. })
  589. it('registrations ride the child fiber: disposing the run removes them; a provider reload mid-run cannot', async () => {
  590. const { ctx, parent, disposeProvider } = await setup([
  591. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 4 }),
  592. ])
  593. expect(ctx.tools.get(STRUCTURED_OUTPUT_TOOL)).toBeUndefined()
  594. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  595. // A backend hot-reload mid-run must not unregister the capture tool out
  596. // from under the live child: the registration rides the CHILD's fiber.
  597. disposeProvider()
  598. const result = await run.result
  599. expect(result.structured).toEqual({ answer: 4 })
  600. const child = ctx.agents.get(run.id)!
  601. expect(ctx.tools.get(STRUCTURED_OUTPUT_TOOL, child)).toBeDefined()
  602. await run.dispose()
  603. // Child disposed ⇒ its scoped registrations are gone.
  604. expect(ctx.tools.get(STRUCTURED_OUTPUT_TOOL, child)).toBeUndefined()
  605. })
  606. })
  607. it('a structured_output call from an agent WITHOUT a structured run is UNKNOWN_TOOL (the tool does not exist for it)', async () => {
  608. const { ctx, parent } = await setup([])
  609. const result = await ctx.tools.execute({
  610. signal: testToolSignal,
  611. callId: 'x' as never,
  612. name: STRUCTURED_OUTPUT_TOOL,
  613. arguments: { answer: 1 },
  614. agent: parent,
  615. })
  616. expect(result.isError).toBe(true)
  617. expect(result.error?.code).toBe('UNKNOWN_TOOL')
  618. })
  619. it('a structured_output call with NO calling agent at all is UNKNOWN_TOOL', async () => {
  620. const { ctx } = await setup([])
  621. const result = await ctx.tools.execute({
  622. signal: testToolSignal,
  623. callId: 'x' as never,
  624. name: STRUCTURED_OUTPUT_TOOL,
  625. arguments: { answer: 1 },
  626. })
  627. expect(result.isError).toBe(true)
  628. expect(result.error?.code).toBe('UNKNOWN_TOOL')
  629. })
  630. it('a failed execution stage is discarded and never promoted by a later call', async () => {
  631. const { ctx, parent } = await setup([
  632. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 1 }),
  633. ])
  634. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  635. // A prepended post-execute listener blocks the first capture without
  636. // delegating. The final-result notification discards that execution's
  637. // stage when it observes the error.
  638. let blocks = 1
  639. ctx.on('tools/post-execute', (exec, _result, next) => {
  640. if (exec.name === STRUCTURED_OUTPUT_TOOL && blocks > 0) {
  641. blocks -= 1
  642. return Promise.resolve({ kind: 'block' as const, feedback: [{ type: 'text' as const, text: 'rejected' }] })
  643. }
  644. return next()
  645. }, { prepend: true })
  646. const result = await run.result
  647. const child = ctx.agents.get(run.id)!
  648. // The blocked capture must NOT surface as structured success…
  649. expect(result.stopReason).toBe('error')
  650. expect(result.structured).toBeUndefined()
  651. // …and a LATER invalid call (its own body staged nothing) must not
  652. // resurrect c1's discarded value: drive the pipeline directly.
  653. const invalid = await ctx.tools.execute({
  654. signal: testToolSignal,
  655. callId: 'c2' as never,
  656. name: STRUCTURED_OUTPUT_TOOL,
  657. arguments: { answer: 'not-a-number' },
  658. agent: child,
  659. })
  660. expect(invalid.isError).toBe(true)
  661. // A fresh valid call still captures ITS OWN value.
  662. const valid = await ctx.tools.execute({
  663. signal: testToolSignal,
  664. callId: 'c3' as never,
  665. name: STRUCTURED_OUTPUT_TOOL,
  666. arguments: { answer: 9 },
  667. agent: child,
  668. })
  669. expect(valid.isError).toBeFalsy()
  670. await run.dispose()
  671. })
  672. it('reusing a failed execution\'s call id never promotes its discarded stage', async () => {
  673. const { ctx, parent } = await setup([
  674. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 1 }),
  675. ])
  676. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  677. // Block the first capture after its body stages a value. Its final error
  678. // discards that execution's stage.
  679. let blocks = 1
  680. ctx.on('tools/post-execute', (exec, _result, next) => {
  681. if (exec.name === STRUCTURED_OUTPUT_TOOL && blocks > 0) {
  682. blocks -= 1
  683. return Promise.resolve({ kind: 'block' as const, feedback: [{ type: 'text' as const, text: 'rejected' }] })
  684. }
  685. return next()
  686. }, { prepend: true })
  687. await run.result
  688. const child = ctx.agents.get(run.id)!
  689. // A SECOND capture call with the SAME call id whose body never stages
  690. // (invalid args throw before the stage): the discarded value must not ride
  691. // its acceptance.
  692. const reused = await ctx.tools.execute({
  693. signal: testToolSignal,
  694. callId: 'c1' as never,
  695. name: STRUCTURED_OUTPUT_TOOL,
  696. arguments: { answer: 'not-a-number' },
  697. agent: child,
  698. })
  699. expect(reused.isError).toBe(true)
  700. // Nothing was ever committed: a fresh valid call is still required.
  701. const valid = await ctx.tools.execute({
  702. signal: testToolSignal,
  703. callId: 'c1' as never,
  704. name: STRUCTURED_OUTPUT_TOOL,
  705. arguments: { answer: 5 },
  706. agent: child,
  707. })
  708. expect(valid.isError).toBeFalsy()
  709. await run.dispose()
  710. })
  711. it('a pre-execute deny with call-id reuse cannot promote another execution\'s stage', async () => {
  712. const { ctx, parent } = await setup([
  713. toolCallResponse('c1', STRUCTURED_OUTPUT_TOOL, { answer: 1 }),
  714. ])
  715. const run = await ctx.subagents.start('spawn', structuredRequest(parent))
  716. // Discard the first capture's stage via a final post-execute block.
  717. let blocks = 1
  718. ctx.on('tools/post-execute', (exec, _result, next) => {
  719. if (exec.name === STRUCTURED_OUTPUT_TOOL && blocks > 0) {
  720. blocks -= 1
  721. return Promise.resolve({ kind: 'block' as const, feedback: [{ type: 'text' as const, text: 'rejected' }] })
  722. }
  723. return next()
  724. }, { prepend: true })
  725. await run.result
  726. const child = ctx.agents.get(run.id)!
  727. // A prepended pre-execute deny skips the body, while the denied call still
  728. // reaches the final notification with the same adapter-minted call id.
  729. const offDeny = ctx.on('tools/pre-execute', (exec) => {
  730. if (exec.name === STRUCTURED_OUTPUT_TOOL) {
  731. return Promise.resolve({ kind: 'deny' as const, reason: 'outer veto' })
  732. }
  733. return undefined as never
  734. }, { prepend: true })
  735. const denied = await ctx.tools.execute({
  736. signal: testToolSignal,
  737. callId: 'c1' as never,
  738. name: STRUCTURED_OUTPUT_TOOL,
  739. arguments: { answer: 2 },
  740. agent: child,
  741. })
  742. expect(denied.isError).toBe(true)
  743. offDeny()
  744. // The discarded value was never promoted: a fresh valid call is required
  745. // (and succeeds, proving the runtime is not wedged).
  746. const valid = await ctx.tools.execute({
  747. signal: testToolSignal,
  748. callId: 'c1' as never,
  749. name: STRUCTURED_OUTPUT_TOOL,
  750. arguments: { answer: 5 },
  751. agent: child,
  752. })
  753. expect(valid.isError).toBeFalsy()
  754. await run.dispose()
  755. })
  756. })