tool-calls.spec.ts 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762
  1. /**
  2. * Exercises scheduler ordering and cancellation with deterministic gated tools.
  3. * ACP expected outputs own transcript-facing coverage.
  4. */
  5. import { describe, expect, it } from 'vitest'
  6. import { Context } from '@deepseek-ai/cordis'
  7. import { createUserMessage, CallId, StreamChunk } from '@deepseek-ai/dsh-llm'
  8. import SessionStore, { SessionEvent, SessionId } from '@deepseek-ai/dsh-session'
  9. import SystemPrompt from '@deepseek-ai/dsh-system-prompt'
  10. import LlmRuntime from '@deepseek-ai/dsh-llm'
  11. import ToolRuntime, { defineContentToolFixture, TOOL_ABORTED_BEFORE_DISPATCH, TOOL_RUNTIME_SCHEDULER, type PostToolDecision, type PreToolDecision } from '@deepseek-ai/dsh-tools'
  12. import AgentRegistry, { type Agent } from '@deepseek-ai/dsh-agent'
  13. import AgentLoop, { DEFAULT_MAX_PARALLEL_TOOL_CALLS } from '@deepseek-ai/dsh-agent-loop'
  14. import { MockAdapter, textResponse } from './mock-adapter.ts'
  15. import { CodeRuntime } from '@deepseek-ai/dsh-code-runtime'
  16. import type { CodeRunRequest, CodeRunResult } from '@deepseek-ai/dsh-code-runtime'
  17. async function harness(adapter: MockAdapter, maxParallelToolCalls?: number) {
  18. const ctx = new Context()
  19. await ctx.plugin(LlmRuntime)
  20. await ctx.plugin(SessionStore)
  21. await ctx.plugin(SystemPrompt, { persona: '' })
  22. await ctx.plugin(ToolRuntime)
  23. await ctx.plugin(AgentRegistry)
  24. await ctx.plugin(AgentLoop, {
  25. agents: [],
  26. ...maxParallelToolCalls === undefined ? {} : { maxParallelToolCalls },
  27. })
  28. ctx.llm.registerAdapter(['mock'], adapter)
  29. return ctx
  30. }
  31. function waitForIdle(ctx: Context, agent: Agent): Promise<void> {
  32. return new Promise((resolve) => {
  33. const dispose = ctx.on('agent/status', ({ agent: subject, status }) => {
  34. if (subject === agent && status === 'idle') { dispose(); resolve() }
  35. })
  36. })
  37. }
  38. function events(agent: Agent): SessionEvent[] {
  39. return [...agent.session.events]
  40. }
  41. /** Build one assistant response containing the supplied tool calls. */
  42. function multiCall(calls: { id: string; name: string; args: object }[]): StreamChunk[] {
  43. const chunks: StreamChunk[] = []
  44. calls.forEach((call, index) => {
  45. chunks.push(
  46. { type: 'block-start', index, blockType: 'tool-call' },
  47. { type: 'block-end', index, block: { type: 'tool-call', id: CallId(call.id), name: call.name, arguments: JSON.stringify(call.args) } },
  48. )
  49. })
  50. chunks.push(
  51. { type: 'usage', usage: { inputTokens: 5, outputTokens: 5 } },
  52. { type: 'finish', reason: { kind: 'tool-calls' } },
  53. )
  54. return chunks
  55. }
  56. /** A tool whose calls block until the test releases them by callId. */
  57. function gatedTool(name: string, parallel: boolean) {
  58. const gates = new Map<string, () => void>()
  59. const started: string[] = []
  60. const tool = defineContentToolFixture({
  61. name,
  62. description: `gated ${name}`,
  63. parameters: { id: { type: 'string', required: true } },
  64. ...parallel ? { isConcurrencySafe: () => true } : {},
  65. async execute(args) {
  66. started.push(args.id)
  67. await new Promise<void>((resolve) => { gates.set(args.id, resolve) })
  68. return [{ type: 'text', text: `done-${args.id}` }]
  69. },
  70. })
  71. return {
  72. tool,
  73. started,
  74. release(id: string) { gates.get(id)?.(); gates.delete(id) },
  75. pending() { return [...gates.keys()] },
  76. }
  77. }
  78. function gatedParallelTool(name: string) {
  79. return gatedTool(name, true)
  80. }
  81. function gatedExclusiveTool(name: string) {
  82. return gatedTool(name, false)
  83. }
  84. /** Poll until `predicate` holds, letting microtasks/timers drain between checks. */
  85. async function until(predicate: () => boolean): Promise<void> {
  86. for (let i = 0; i < 1000 && !predicate(); i++) await new Promise(r => setTimeout(r, 0))
  87. if (!predicate()) throw new Error('until: condition never held')
  88. }
  89. describe('tool-call scheduler: grouping and barriers', () => {
  90. it('runs parallel-safe siblings concurrently (all start before any completes)', async () => {
  91. const adapter = new MockAdapter([
  92. multiCall([{ id: 'c1', name: 'p', args: { id: '1' } }, { id: 'c2', name: 'p', args: { id: '2' } }, { id: 'c3', name: 'p', args: { id: '3' } }]),
  93. textResponse('done'),
  94. ])
  95. const ctx = await harness(adapter)
  96. const gated = gatedParallelTool('p')
  97. ctx.tools.register(gated.tool)
  98. const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
  99. agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
  100. await until(() => gated.started.length === 3)
  101. expect(gated.started).toEqual(['1', '2', '3'])
  102. gated.release('1'); gated.release('2'); gated.release('3')
  103. await waitForIdle(ctx, agent)
  104. })
  105. it('an exclusive call between two parallel-safe calls forms a barrier (3 groups)', async () => {
  106. const order: string[] = []
  107. const adapter = new MockAdapter([
  108. multiCall([
  109. { id: 'c1', name: 'r', args: { id: 'A1' } },
  110. { id: 'c2', name: 'w', args: { id: 'A2' } },
  111. { id: 'c3', name: 'r', args: { id: 'A3' } },
  112. ]),
  113. textResponse('done'),
  114. ])
  115. const ctx = await harness(adapter)
  116. ctx.tools.register(defineContentToolFixture({
  117. name: 'r', description: 'read', parameters: { id: { type: 'string', required: true } },
  118. isConcurrencySafe: () => true,
  119. async execute(args) { order.push(`r-start-${args.id}`); order.push(`r-end-${args.id}`); return [{ type: 'text', text: 'r' }] },
  120. }))
  121. ctx.tools.register(defineContentToolFixture({
  122. name: 'w', description: 'write', parameters: { id: { type: 'string', required: true } },
  123. async execute(args) { order.push(`w-${args.id}`); return [{ type: 'text', text: 'w' }] },
  124. }))
  125. const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
  126. agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
  127. await waitForIdle(ctx, agent)
  128. expect(order).toEqual(['r-start-A1', 'r-end-A1', 'w-A2', 'r-start-A3', 'r-end-A3'])
  129. })
  130. it('reclassifies pending calls after an exclusive barrier replaces their tool', async () => {
  131. const adapter = new MockAdapter([
  132. multiCall([
  133. { id: 'c1', name: 'replace', args: { id: '0' } },
  134. { id: 'c2', name: 'x', args: { id: '1' } },
  135. { id: 'c3', name: 'x', args: { id: '2' } },
  136. ]),
  137. textResponse('done'),
  138. ])
  139. const ctx = await harness(adapter)
  140. const replacement = gatedExclusiveTool('x')
  141. const disposeSafe = ctx.tools.register(defineContentToolFixture({
  142. name: 'x',
  143. description: 'initially safe',
  144. parameters: { id: { type: 'string', required: true } },
  145. isConcurrencySafe: () => true,
  146. async execute(args) { return [{ type: 'text', text: `old-${args.id}` }] },
  147. }))
  148. ctx.tools.register(defineContentToolFixture({
  149. name: 'replace',
  150. description: 'replace x',
  151. parameters: { id: { type: 'string', required: true } },
  152. async execute() {
  153. disposeSafe()
  154. ctx.tools.register(replacement.tool)
  155. return [{ type: 'text', text: 'replaced' }]
  156. },
  157. }))
  158. const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
  159. agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
  160. await until(() => replacement.started.length === 1)
  161. await new Promise(r => setTimeout(r, 5))
  162. expect(replacement.started).toEqual(['1'])
  163. replacement.release('1')
  164. await until(() => replacement.started.length === 2)
  165. expect(replacement.started).toEqual(['1', '2'])
  166. replacement.release('2')
  167. await waitForIdle(ctx, agent)
  168. })
  169. it('stops replenishing when a result observer makes the next call exclusive', async () => {
  170. const adapter = new MockAdapter([
  171. multiCall([
  172. { id: 'c1', name: 'x', args: { id: '1' } },
  173. { id: 'c2', name: 'x', args: { id: '2' } },
  174. { id: 'c3', name: 'x', args: { id: '3' } },
  175. ]),
  176. textResponse('done'),
  177. ])
  178. const ctx = await harness(adapter, 2)
  179. const initial = gatedParallelTool('x')
  180. const replacement = gatedExclusiveTool('x')
  181. const disposeInitial = ctx.tools.register(initial.tool)
  182. ctx.on('tools/result', (exec) => {
  183. if (exec.callId !== CallId('c1')) return
  184. disposeInitial()
  185. ctx.tools.register(replacement.tool)
  186. })
  187. const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
  188. agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
  189. await until(() => initial.started.length === 2)
  190. initial.release('1')
  191. await until(() => events(agent).some(event =>
  192. event.type === 'tool/result' && event.data.message.source.callId === CallId('c1')))
  193. await new Promise(r => setTimeout(r, 5))
  194. expect(replacement.started).toEqual([])
  195. initial.release('2')
  196. await until(() => replacement.started.length === 1)
  197. expect(replacement.started).toEqual(['3'])
  198. replacement.release('3')
  199. await waitForIdle(ctx, agent)
  200. })
  201. })
  202. describe('tool-call scheduler: model-order results despite out-of-order settlement', () => {
  203. it('commits tool/result in model order even when a later call settles first', async () => {
  204. const adapter = new MockAdapter([
  205. multiCall([{ id: 'c1', name: 'p', args: { id: '1' } }, { id: 'c2', name: 'p', args: { id: '2' } }]),
  206. textResponse('done'),
  207. ])
  208. const ctx = await harness(adapter)
  209. const gated = gatedParallelTool('p')
  210. ctx.tools.register(gated.tool)
  211. const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
  212. agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
  213. await until(() => gated.started.length === 2)
  214. gated.release('2')
  215. await new Promise(r => setTimeout(r, 5))
  216. const beforeFirst = events(agent).filter(e => e.type === 'tool/result')
  217. expect(beforeFirst).toEqual([])
  218. gated.release('1')
  219. await waitForIdle(ctx, agent)
  220. const results = events(agent).filter(e => e.type === 'tool/result')
  221. expect(results.map(e => e.data.message.source.callId)).toEqual([CallId('c1'), CallId('c2')])
  222. })
  223. it('derived history pairs calls in model order regardless of tool/call log interleaving', async () => {
  224. const adapter = new MockAdapter([
  225. multiCall([{ id: 'c1', name: 'p', args: { id: '1' } }, { id: 'c2', name: 'p', args: { id: '2' } }]),
  226. textResponse('done'),
  227. ])
  228. const ctx = await harness(adapter)
  229. const gated = gatedParallelTool('p')
  230. ctx.tools.register(gated.tool)
  231. const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
  232. agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
  233. await until(() => gated.started.length === 2)
  234. gated.release('2'); gated.release('1')
  235. await waitForIdle(ctx, agent)
  236. const messages = agent.session.deriveMessages()
  237. const toolResults = messages.flatMap(m => m.content.filter(b => b.type === 'tool-result'))
  238. expect(toolResults.map(b => b.toolCallId)).toEqual([CallId('c1'), CallId('c2')])
  239. })
  240. })
  241. describe('tool-call scheduler: rolling pool honors maxParallelToolCalls', () => {
  242. it('rejects invalid global maxParallelToolCalls config at plugin load', async () => {
  243. await expect(harness(new MockAdapter([]), 0)).rejects.toThrow()
  244. await expect(harness(new MockAdapter([]), 1.5)).rejects.toThrow()
  245. })
  246. it('defensively rejects invalid caps when direct construction bypasses the config schema', () => {
  247. expect(() => new AgentLoop(new Context(), { agents: [], maxParallelToolCalls: 0 }))
  248. .toThrow('maxParallelToolCalls must be a positive integer')
  249. expect(() => new AgentLoop(new Context(), { agents: [], maxParallelToolCalls: 1.5 }))
  250. .toThrow('maxParallelToolCalls must be a positive integer')
  251. })
  252. it('defaults the cap when direct construction bypasses the config schema', async () => {
  253. const ctx = new Context()
  254. await ctx.plugin(LlmRuntime)
  255. await ctx.plugin(SessionStore)
  256. await ctx.plugin(SystemPrompt, { persona: '' })
  257. await ctx.plugin(ToolRuntime)
  258. await ctx.plugin(AgentRegistry)
  259. const loop = new AgentLoop(ctx, { agents: [] })
  260. expect(loop.config.maxParallelToolCalls).toBe(DEFAULT_MAX_PARALLEL_TOOL_CALLS)
  261. await ctx.fiber.dispose()
  262. })
  263. it('starts at most the cap, replenishing as calls settle', async () => {
  264. const adapter = new MockAdapter([
  265. multiCall([1, 2, 3, 4].map(n => ({ id: `c${n}`, name: 'p', args: { id: String(n) } }))),
  266. textResponse('done'),
  267. ])
  268. const ctx = await harness(adapter, 2)
  269. const gated = gatedParallelTool('p')
  270. ctx.tools.register(gated.tool)
  271. const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
  272. agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
  273. await until(() => gated.started.length === 2)
  274. await new Promise(r => setTimeout(r, 5))
  275. expect(gated.started).toEqual(['1', '2'])
  276. gated.release('1')
  277. await until(() => gated.started.length === 3)
  278. expect(gated.started).toEqual(['1', '2', '3'])
  279. expect(events(agent)
  280. .filter(e => e.type === 'tool/call' || e.type === 'tool/result')
  281. .map(e => e.type === 'tool/call'
  282. ? `${e.type}:${String(e.data.callId)}`
  283. : `${e.type}:${String(e.data.message.source.callId)}`)
  284. .slice(0, 4))
  285. .toEqual(['tool/call:c1', 'tool/call:c2', 'tool/result:c1', 'tool/call:c3'])
  286. gated.release('2'); gated.release('3')
  287. await until(() => gated.started.length === 4)
  288. gated.release('4')
  289. await waitForIdle(ctx, agent)
  290. expect(events(agent).filter(e => e.type === 'tool/result').map(e => e.data.message.source.callId))
  291. .toEqual([CallId('c1'), CallId('c2'), CallId('c3'), CallId('c4')])
  292. })
  293. it('maxParallelToolCalls: 1 is fully serial (no second start before the first settles)', async () => {
  294. const adapter = new MockAdapter([
  295. multiCall([{ id: 'c1', name: 'p', args: { id: '1' } }, { id: 'c2', name: 'p', args: { id: '2' } }]),
  296. textResponse('done'),
  297. ])
  298. const ctx = await harness(adapter, 1)
  299. const gated = gatedParallelTool('p')
  300. ctx.tools.register(gated.tool)
  301. const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
  302. agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
  303. await until(() => gated.started.length === 1)
  304. await new Promise(r => setTimeout(r, 5))
  305. expect(gated.started).toEqual(['1'])
  306. gated.release('1')
  307. await until(() => gated.started.length === 2)
  308. gated.release('2')
  309. await waitForIdle(ctx, agent)
  310. })
  311. it('applies the configured cap to every factory-created agent', async () => {
  312. const adapter = new MockAdapter([
  313. multiCall([{ id: 'c1', name: 'p', args: { id: '1' } }, { id: 'c2', name: 'p', args: { id: '2' } }]),
  314. textResponse('done'),
  315. ])
  316. const ctx = new Context()
  317. await ctx.plugin(LlmRuntime)
  318. await ctx.plugin(SessionStore)
  319. await ctx.plugin(SystemPrompt, { persona: '' })
  320. await ctx.plugin(ToolRuntime)
  321. await ctx.plugin(AgentRegistry)
  322. await ctx.plugin(AgentLoop, { agents: [], maxParallelToolCalls: 1 })
  323. ctx.llm.registerAdapter(['mock'], adapter)
  324. const gated = gatedParallelTool('p')
  325. ctx.tools.register(gated.tool)
  326. const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
  327. agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
  328. await until(() => gated.started.length === 1)
  329. await new Promise(r => setTimeout(r, 5))
  330. expect(gated.started).toEqual(['1'])
  331. gated.release('1')
  332. await until(() => gated.started.length === 2)
  333. gated.release('2')
  334. await waitForIdle(ctx, agent)
  335. })
  336. })
  337. describe('tool-call scheduler: ordered middleware and additional contexts', () => {
  338. it('tools/pre-execute and tools/post-execute observe model call order', async () => {
  339. const adapter = new MockAdapter([
  340. multiCall([{ id: 'c1', name: 'p', args: { id: '1' } }, { id: 'c2', name: 'p', args: { id: '2' } }, { id: 'c3', name: 'p', args: { id: '3' } }]),
  341. textResponse('done'),
  342. ])
  343. const ctx = await harness(adapter)
  344. const gated = gatedParallelTool('p')
  345. ctx.tools.register(gated.tool)
  346. const pre: string[] = []
  347. const post: string[] = []
  348. ctx.on('tools/pre-execute', async (exec, next): Promise<PreToolDecision> => { pre.push(String(exec.callId)); return next() })
  349. ctx.on('tools/post-execute', async (exec, _result, next): Promise<PostToolDecision> => { post.push(String(exec.callId)); return next() })
  350. const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
  351. agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
  352. await until(() => gated.started.length === 3)
  353. gated.release('3'); gated.release('2'); gated.release('1')
  354. await waitForIdle(ctx, agent)
  355. expect(pre).toEqual([CallId('c1'), CallId('c2'), CallId('c3')].map(String))
  356. expect(post).toEqual([CallId('c1'), CallId('c2'), CallId('c3')].map(String))
  357. })
  358. it('injects additional contexts in model call order, not settlement order', async () => {
  359. const adapter = new MockAdapter([
  360. multiCall([{ id: 'c1', name: 'p', args: { id: '1' } }, { id: 'c2', name: 'p', args: { id: '2' } }]),
  361. textResponse('done'),
  362. ])
  363. const ctx = await harness(adapter, 2)
  364. const gated = gatedParallelTool('p')
  365. ctx.tools.register(gated.tool)
  366. ctx.on('tools/post-execute', async (exec, _result): Promise<PostToolDecision> =>
  367. ({ kind: 'accept', additionalContexts: [createUserMessage({
  368. content: [{ type: 'text', text: `ctx-${exec.callId}` }], source: { kind: 'plugin', plugin: 'p' },
  369. })] }))
  370. const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
  371. agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
  372. await until(() => gated.started.length === 2)
  373. gated.release('2'); gated.release('1')
  374. await waitForIdle(ctx, agent)
  375. const log = events(agent)
  376. const contextTexts = log.filter(e => e.type === 'user/message' && e.data.source.kind === 'plugin')
  377. .map(e => ((e.data as { content: { text: string }[] }).content[0]!).text)
  378. expect(contextTexts).toEqual(['ctx-c1', 'ctx-c2'])
  379. const lastResult = log.findLastIndex(e => e.type === 'tool/result')
  380. const firstContext = log.findIndex(e => e.type === 'user/message' && e.data.source.kind === 'plugin')
  381. expect(lastResult).toBeLessThan(firstContext)
  382. })
  383. it('orders pre-execute denials and errors without dispatching them', async () => {
  384. const adapter = new MockAdapter([
  385. multiCall([
  386. { id: 'c1', name: 'p', args: { id: '1' } },
  387. { id: 'c2', name: 'p', args: { id: '2' } },
  388. { id: 'c3', name: 'p', args: { id: '3' } },
  389. ]),
  390. textResponse('done'),
  391. ])
  392. const ctx = await harness(adapter)
  393. const gated = gatedParallelTool('p')
  394. ctx.tools.register(gated.tool)
  395. const post: string[] = []
  396. ctx.on('tools/pre-execute', async (exec, next): Promise<PreToolDecision> => {
  397. if (exec.callId === CallId('c2')) return { kind: 'deny', reason: 'blocked by policy' }
  398. if (exec.callId === CallId('c3')) throw new Error('pre exploded')
  399. return next()
  400. })
  401. ctx.on('tools/post-execute', async (exec, _result, next): Promise<PostToolDecision> => {
  402. post.push(String(exec.callId))
  403. return next()
  404. })
  405. const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
  406. agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
  407. await until(() => gated.started.length === 1)
  408. gated.release('1')
  409. await waitForIdle(ctx, agent)
  410. expect(gated.started).toEqual(['1'])
  411. expect(post).toEqual(['c1', 'c2'])
  412. const results = events(agent).filter(e => e.type === 'tool/result')
  413. expect(results.map(e => e.data.message.source.callId)).toEqual([CallId('c1'), CallId('c2'), CallId('c3')])
  414. expect((results[1]!.data.message.content[0].content[0] as { text: string }).text).toContain('blocked by policy')
  415. expect((results[2]!.data.message.content[0].content[0] as { text: string }).text).toContain('pre exploded')
  416. })
  417. })
  418. describe('tool-call scheduler: abort handling', () => {
  419. it('starts no calls when the signal is already aborted before a parallel group', async () => {
  420. const adapter = new MockAdapter([
  421. multiCall([{ id: 'c1', name: 'p', args: { id: '1' } }, { id: 'c2', name: 'p', args: { id: '2' } }]),
  422. textResponse('should never be requested'),
  423. ])
  424. const ctx = await harness(adapter)
  425. const gated = gatedParallelTool('p')
  426. ctx.tools.register(gated.tool)
  427. const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
  428. ctx.on('session/event', (session, event) => {
  429. if (session === agent.session && event.type === 'assistant/message') {
  430. agent.cancel({ kind: 'user' })
  431. }
  432. })
  433. agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
  434. await waitForIdle(ctx, agent)
  435. expect(gated.started).toEqual([])
  436. expect(events(agent).filter(e => e.type === 'tool/call').map(e => e.data.callId))
  437. .toEqual([CallId('c1'), CallId('c2')])
  438. expect(events(agent).filter(e => e.type === 'tool/result').map(e => ({
  439. callId: e.data.message.source.callId,
  440. isError: e.data.message.content[0].isError,
  441. error: e.data.error,
  442. }))).toEqual([
  443. { callId: CallId('c1'), isError: true, error: { name: 'AbortError', code: TOOL_ABORTED_BEFORE_DISPATCH } },
  444. { callId: CallId('c2'), isError: true, error: { name: 'AbortError', code: TOOL_ABORTED_BEFORE_DISPATCH } },
  445. ])
  446. })
  447. it('skips dispatch and stops starting siblings when abort fires during ordered pre-execute', async () => {
  448. const adapter = new MockAdapter([
  449. multiCall([{ id: 'c1', name: 'p', args: { id: '1' } }, { id: 'c2', name: 'p', args: { id: '2' } }]),
  450. textResponse('should never be requested'),
  451. ])
  452. const ctx = await harness(adapter)
  453. const gated = gatedParallelTool('p')
  454. ctx.tools.register(gated.tool)
  455. const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
  456. ctx.on('tools/pre-execute', async (exec, next): Promise<PreToolDecision> => {
  457. if (exec.callId === CallId('c1')) {
  458. agent.cancel({ kind: 'user' })
  459. }
  460. return next()
  461. })
  462. agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
  463. await waitForIdle(ctx, agent)
  464. expect(gated.started).toEqual([])
  465. expect(events(agent).filter(e => e.type === 'tool/call').map(e => e.data.callId))
  466. .toEqual([CallId('c1'), CallId('c2')])
  467. expect(events(agent).filter(e => e.type === 'tool/result').map(e => ({
  468. callId: e.data.message.source.callId,
  469. isError: e.data.message.content[0].isError,
  470. error: e.data.error,
  471. }))).toEqual([
  472. { callId: CallId('c1'), isError: true, error: { name: 'AbortError', code: TOOL_ABORTED_BEFORE_DISPATCH } },
  473. { callId: CallId('c2'), isError: true, error: { name: 'AbortError', code: TOOL_ABORTED_BEFORE_DISPATCH } },
  474. ])
  475. })
  476. it('stops replenishing after abort, commits started results, and parks accepted additional contexts', async () => {
  477. const adapter = new MockAdapter([
  478. multiCall([1, 2, 3, 4].map(n => ({ id: `c${n}`, name: 'p', args: { id: String(n) } }))),
  479. textResponse('after wake'),
  480. ])
  481. const ctx = await harness(adapter, 2)
  482. const gated = gatedParallelTool('p')
  483. ctx.tools.register(gated.tool)
  484. ctx.on('tools/post-execute', async (exec, _result, next): Promise<PostToolDecision> => ({
  485. ...await next(),
  486. additionalContexts: [createUserMessage({
  487. content: [{ type: 'text', text: `ctx-${exec.callId}` }], source: { kind: 'plugin', plugin: 'p' },
  488. })],
  489. }))
  490. const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
  491. agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
  492. await until(() => gated.started.length === 2)
  493. agent.cancel({ kind: 'user' })
  494. gated.release('1')
  495. gated.release('2')
  496. await waitForIdle(ctx, agent)
  497. expect(gated.started).toEqual(['1', '2'])
  498. expect(events(agent).filter(e => e.type === 'tool/call').map(e => e.data.callId))
  499. .toEqual([CallId('c1'), CallId('c2'), CallId('c3'), CallId('c4')])
  500. expect(events(agent).filter(e => e.type === 'tool/result').map(e => e.data.message.source.callId))
  501. .toEqual([CallId('c1'), CallId('c2'), CallId('c3'), CallId('c4')])
  502. expect(events(agent).filter(e => e.type === 'tool/result').slice(-2).map(e => ({
  503. callId: e.data.message.source.callId,
  504. isError: e.data.message.content[0].isError,
  505. error: e.data.error,
  506. })))
  507. .toEqual([
  508. {
  509. callId: CallId('c3'),
  510. isError: true,
  511. error: { name: 'AbortError', code: TOOL_ABORTED_BEFORE_DISPATCH },
  512. },
  513. {
  514. callId: CallId('c4'),
  515. isError: true,
  516. error: { name: 'AbortError', code: TOOL_ABORTED_BEFORE_DISPATCH },
  517. },
  518. ])
  519. const settled = events(agent).filter(e => e.type === 'tool/result'
  520. || (e.type === 'user/message' && e.data.source.kind === 'plugin'))
  521. expect(settled.map(e => e.type))
  522. .toEqual(['tool/result', 'tool/result', 'tool/result', 'tool/result'])
  523. expect(agent.inbox.nextStep.map(message => message.content[0]))
  524. .toEqual([
  525. { type: 'text', text: 'ctx-c1' },
  526. { type: 'text', text: 'ctx-c2' },
  527. ])
  528. const idle = waitForIdle(ctx, agent)
  529. agent.followup(createUserMessage({ content: [{ type: 'text', text: 'wake' }], source: { kind: 'user' } }))
  530. await idle
  531. expect(events(agent).flatMap(e =>
  532. e.type === 'user/message'
  533. && e.data.source.kind === 'plugin'
  534. && e.data.content[0]?.type === 'text'
  535. ? [e.data.content[0].text]
  536. : []))
  537. .toEqual(['ctx-c1', 'ctx-c2'])
  538. })
  539. it('does not run an exclusive barrier after a parallel group aborts', async () => {
  540. const adapter = new MockAdapter([
  541. multiCall([
  542. { id: 'c1', name: 'p', args: { id: '1' } },
  543. { id: 'c2', name: 'p', args: { id: '2' } },
  544. { id: 'c3', name: 'x', args: { id: '3' } },
  545. ]),
  546. textResponse('should never be requested'),
  547. ])
  548. const ctx = await harness(adapter, 2)
  549. const gated = gatedParallelTool('p')
  550. const exclusive: string[] = []
  551. ctx.tools.register(gated.tool)
  552. ctx.tools.register(defineContentToolFixture({
  553. name: 'x',
  554. description: 'exclusive',
  555. parameters: { id: { type: 'string', required: true } },
  556. async execute(args) { exclusive.push(args.id); return [{ type: 'text', text: 'x' }] },
  557. }))
  558. const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
  559. agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
  560. await until(() => gated.started.length === 2)
  561. agent.cancel({ kind: 'user' })
  562. gated.release('1')
  563. gated.release('2')
  564. await waitForIdle(ctx, agent)
  565. expect(exclusive).toEqual([])
  566. expect(events(agent).filter(e => e.type === 'tool/call').map(e => e.data.callId))
  567. .toEqual([CallId('c1'), CallId('c2'), CallId('c3')])
  568. expect(events(agent).filter(e => e.type === 'tool/result').at(-1)?.data)
  569. .toMatchObject({
  570. message: {
  571. source: { kind: 'tool', callId: CallId('c3') },
  572. content: [{ isError: true }],
  573. },
  574. error: { name: 'AbortError', code: TOOL_ABORTED_BEFORE_DISPATCH },
  575. })
  576. })
  577. })
  578. describe('tool-call scheduler: failure quiescence', () => {
  579. it('stops new dispatches and drains started bodies before surfacing the first failure', async () => {
  580. const adapter = new MockAdapter([
  581. multiCall([
  582. { id: 'c1', name: 'p', args: { id: '1' } },
  583. { id: 'c2', name: 'p', args: { id: '2' } },
  584. { id: 'c3', name: 'p', args: { id: '3' } },
  585. ]),
  586. ])
  587. const ctx = await harness(adapter, 3)
  588. const gated = gatedParallelTool('p')
  589. ctx.tools.register(gated.tool)
  590. // The registry contains expected failures as results; replace its internal
  591. // view only to inject the invariant violation this boundary must contain.
  592. const scheduler = ctx.tools[TOOL_RUNTIME_SCHEDULER]
  593. const prepare = scheduler.prepare.bind(scheduler)
  594. const dispatch = scheduler.dispatch.bind(scheduler)
  595. const prepareGate = Promise.withResolvers<undefined>()
  596. let thirdPrepareEntered = false
  597. scheduler.prepare = async (exec) => {
  598. const prepared = await prepare(exec)
  599. if (exec.callId === CallId('c3')) {
  600. thirdPrepareEntered = true
  601. await prepareGate.promise
  602. }
  603. return prepared
  604. }
  605. const schedulerError = new Error('scheduler exploded')
  606. const drainedError = new Error('sibling failed while draining')
  607. let rejectFirst: ((error: Error) => void) | undefined
  608. scheduler.dispatch = exec => exec.callId === CallId('c1')
  609. ? new Promise((_resolve, reject) => { rejectFirst = reject })
  610. : dispatch(exec).then(() => { throw drainedError })
  611. const agent = ctx.agentLoop.create(SessionId('scheduler-failure'), { provider: 'mock', model: 'mock' })
  612. let idle = false
  613. const idlePromise = waitForIdle(ctx, agent).then(() => { idle = true })
  614. agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
  615. await until(() => gated.started.includes('2') && thirdPrepareEntered && rejectFirst !== undefined)
  616. rejectFirst?.(schedulerError)
  617. await new Promise<void>(resolve => setImmediate(resolve))
  618. prepareGate.resolve(undefined)
  619. await new Promise<void>(resolve => setImmediate(resolve))
  620. const startedBeforeDrain = [...gated.started]
  621. const idleBeforeDrain = idle
  622. const turnEndBeforeDrain = events(agent).find(event => event.type === 'turn/end')
  623. for (const id of gated.pending()) gated.release(id)
  624. await idlePromise
  625. expect(startedBeforeDrain).toEqual(['2'])
  626. expect(idleBeforeDrain).toBe(false)
  627. expect(turnEndBeforeDrain).toBeUndefined()
  628. expect(gated.pending()).toEqual([])
  629. expect(events(agent).findLast(event => event.type === 'turn/end')).toMatchObject({
  630. data: { reason: { kind: 'error', error: { message: schedulerError.message, code: 'UNKNOWN' } } },
  631. })
  632. })
  633. })
  634. describe('code-mode native-tool denial through the agent loop', () => {
  635. /** A minimal in-process code runtime for test purposes — never actually runs. */
  636. class FakeCodeRuntime extends CodeRuntime {
  637. readonly language = 'typescript'
  638. readonly isolation = 'fake' as const
  639. async run(_request: CodeRunRequest): Promise<CodeRunResult> {
  640. return { logs: [] }
  641. }
  642. }
  643. async function codeModeHarness(adapter: MockAdapter) {
  644. const ctx = new Context()
  645. await ctx.plugin(LlmRuntime)
  646. await ctx.plugin(SessionStore)
  647. await ctx.plugin(SystemPrompt, { persona: '' })
  648. await ctx.plugin(ToolRuntime, { mode: 'code' })
  649. // eslint-disable-next-line @typescript-eslint/no-explicit-any -- FakeCodeRuntime is an internal test helper with an opaque type shape
  650. await ctx.plugin(FakeCodeRuntime as any)
  651. await ctx.plugin(AgentRegistry)
  652. await ctx.plugin(AgentLoop, { agents: [] })
  653. ctx.llm.registerAdapter(['mock'], adapter)
  654. return ctx
  655. }
  656. it('denies a model-direct native-tool call under code mode: tool body never runs and session records UNKNOWN_TOOL', async () => {
  657. let toolInvoked = false
  658. const tool = defineContentToolFixture({
  659. name: 'write',
  660. description: 'Write a file.',
  661. parameters: {
  662. file_path: { type: 'string', required: true },
  663. content: { type: 'string', required: true },
  664. },
  665. async execute(_args, _exec) {
  666. toolInvoked = true
  667. return [{ type: 'text', text: 'written' }]
  668. },
  669. })
  670. // Scripted model emits a native tool call under code mode — the wire
  671. // never advertised it, but a non-compliant provider may still emit one.
  672. const adapter = new MockAdapter([
  673. [
  674. ...multiCall([{ id: 'call-1', name: 'write', args: { file_path: '/tmp/test', content: 'hello' } }]),
  675. ...textResponse('ok'),
  676. ],
  677. ])
  678. const ctx = await codeModeHarness(adapter)
  679. ctx.tools.register(tool)
  680. const agent = ctx.agentLoop.create(SessionId('code-native'), { provider: 'mock', model: 'mock' })
  681. agent.followup(createUserMessage({ content: [{ type: 'text', text: 'write a file' }], source: { kind: 'user' } }))
  682. await waitForIdle(ctx, agent)
  683. // The tool body must NOT have executed — the collapse denied the call
  684. // at createExecution, before the body could start.
  685. expect(toolInvoked).toBe(false)
  686. // The session must record a tool/result with UNKNOWN_TOOL error so the
  687. // transcript faithfully captures that the call was denied.
  688. const sessionEvents = events(agent)
  689. const toolResult = sessionEvents.find(e => e.type === 'tool/result')
  690. expect(toolResult).toBeDefined()
  691. expect(toolResult!.data.error).toMatchObject({
  692. name: 'ToolNotFoundError',
  693. code: 'UNKNOWN_TOOL',
  694. })
  695. })
  696. })