tool-calls.spec.ts 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769
  1. /**
  2. * Exercises scheduler ordering and cancellation with deterministic gated tools.
  3. * ACP expected outputs own transcript-facing coverage.
  4. */
  5. import { describe, expect, it } from 'vitest'
  6. import { Context } from '@deepseek-ai/cordis'
  7. import { createUserMessage, ToolCallId, StreamChunk } from '@deepseek-ai/dsh-llm'
  8. import SessionStore, { SessionEvent, SessionId } from '@deepseek-ai/dsh-session'
  9. import SystemPrompt from '@deepseek-ai/dsh-system-prompt'
  10. import LlmRuntime from '@deepseek-ai/dsh-llm'
  11. import ToolRuntime, { defineContentToolFixture, TOOL_ABORTED_BEFORE_DISPATCH, TOOL_RUNTIME_SCHEDULER, type PostToolDecision, type PreToolDecision } from '@deepseek-ai/dsh-tools'
  12. import AgentRegistry, { type Agent } from '@deepseek-ai/dsh-agent'
  13. import AgentLoop, { DEFAULT_MAX_PARALLEL_TOOL_CALLS } from '@deepseek-ai/dsh-agent-loop'
  14. import SessionProjectionRegistry from '@deepseek-ai/dsh-session-projection'
  15. import { MockAdapter, textResponse } from './mock-adapter.ts'
  16. import { CodeRuntime } from '@deepseek-ai/dsh-code-runtime'
  17. import type { CodeRunRequest, CodeRunResult } from '@deepseek-ai/dsh-code-runtime'
  18. async function harness(adapter: MockAdapter, maxParallelToolCalls?: number) {
  19. const ctx = new Context()
  20. await ctx.plugin(LlmRuntime)
  21. await ctx.plugin(SessionStore)
  22. await ctx.plugin(SessionProjectionRegistry)
  23. await ctx.plugin(SystemPrompt, { personaPrefix: '' })
  24. await ctx.plugin(ToolRuntime)
  25. await ctx.plugin(AgentRegistry)
  26. await ctx.plugin(AgentLoop, {
  27. agents: [],
  28. ...maxParallelToolCalls === undefined ? {} : { maxParallelToolCalls },
  29. })
  30. ctx.llm.registerAdapter(['mock'], adapter)
  31. return ctx
  32. }
  33. function waitForIdle(ctx: Context, agent: Agent): Promise<void> {
  34. return new Promise((resolve) => {
  35. const dispose = ctx.on('agent/status', ({ agent: subject, status }) => {
  36. if (subject === agent && status === 'idle') { dispose(); resolve() }
  37. })
  38. })
  39. }
  40. function events(agent: Agent): readonly SessionEvent[] {
  41. return agent.session.snapshotEvents()
  42. }
  43. /** Build one assistant response containing the supplied tool calls. */
  44. function multiCall(calls: { id: string; name: string; args: object }[]): StreamChunk[] {
  45. const chunks: StreamChunk[] = []
  46. calls.forEach((call, index) => {
  47. chunks.push(
  48. { type: 'block-start', index, blockType: 'tool-call' },
  49. { type: 'block-end', index, block: { type: 'tool-call', id: ToolCallId(call.id), name: call.name, arguments: JSON.stringify(call.args) } },
  50. )
  51. })
  52. chunks.push(
  53. { type: 'usage', usage: { inputTokens: 5, outputTokens: 5 } },
  54. { type: 'finish', reason: { kind: 'tool-calls' } },
  55. )
  56. return chunks
  57. }
  58. /** A tool whose calls block until the test releases them by callId. */
  59. function gatedTool(name: string, parallel: boolean) {
  60. const gates = new Map<string, () => void>()
  61. const started: string[] = []
  62. const tool = defineContentToolFixture({
  63. name,
  64. description: `gated ${name}`,
  65. parameters: { id: { type: 'string', required: true } },
  66. ...parallel ? { isConcurrencySafe: () => true } : {},
  67. async execute(args) {
  68. started.push(args.id)
  69. await new Promise<void>((resolve) => { gates.set(args.id, resolve) })
  70. return [{ type: 'text', text: `done-${args.id}` }]
  71. },
  72. })
  73. return {
  74. tool,
  75. started,
  76. release(id: string) { gates.get(id)?.(); gates.delete(id) },
  77. pending() { return [...gates.keys()] },
  78. }
  79. }
  80. function gatedParallelTool(name: string) {
  81. return gatedTool(name, true)
  82. }
  83. function gatedExclusiveTool(name: string) {
  84. return gatedTool(name, false)
  85. }
  86. /** Poll until `predicate` holds, letting microtasks/timers drain between checks. */
  87. async function until(predicate: () => boolean): Promise<void> {
  88. for (let i = 0; i < 1000 && !predicate(); i++) await new Promise(r => setTimeout(r, 0))
  89. if (!predicate()) throw new Error('until: condition never held')
  90. }
  91. describe('tool-call scheduler: grouping and barriers', () => {
  92. it('runs parallel-safe siblings concurrently (all start before any completes)', async () => {
  93. const adapter = new MockAdapter([
  94. multiCall([{ id: 'c1', name: 'p', args: { id: '1' } }, { id: 'c2', name: 'p', args: { id: '2' } }, { id: 'c3', name: 'p', args: { id: '3' } }]),
  95. textResponse('done'),
  96. ])
  97. const ctx = await harness(adapter)
  98. const gated = gatedParallelTool('p')
  99. ctx.tools.register(gated.tool)
  100. const agent = await ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
  101. agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
  102. await until(() => gated.started.length === 3)
  103. expect(gated.started).toEqual(['1', '2', '3'])
  104. gated.release('1'); gated.release('2'); gated.release('3')
  105. await waitForIdle(ctx, agent)
  106. })
  107. it('an exclusive call between two parallel-safe calls forms a barrier (3 groups)', async () => {
  108. const order: string[] = []
  109. const adapter = new MockAdapter([
  110. multiCall([
  111. { id: 'c1', name: 'r', args: { id: 'A1' } },
  112. { id: 'c2', name: 'w', args: { id: 'A2' } },
  113. { id: 'c3', name: 'r', args: { id: 'A3' } },
  114. ]),
  115. textResponse('done'),
  116. ])
  117. const ctx = await harness(adapter)
  118. ctx.tools.register(defineContentToolFixture({
  119. name: 'r', description: 'read', parameters: { id: { type: 'string', required: true } },
  120. isConcurrencySafe: () => true,
  121. async execute(args) { order.push(`r-start-${args.id}`); order.push(`r-end-${args.id}`); return [{ type: 'text', text: 'r' }] },
  122. }))
  123. ctx.tools.register(defineContentToolFixture({
  124. name: 'w', description: 'write', parameters: { id: { type: 'string', required: true } },
  125. async execute(args) { order.push(`w-${args.id}`); return [{ type: 'text', text: 'w' }] },
  126. }))
  127. const agent = await ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
  128. agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
  129. await waitForIdle(ctx, agent)
  130. expect(order).toEqual(['r-start-A1', 'r-end-A1', 'w-A2', 'r-start-A3', 'r-end-A3'])
  131. })
  132. it('reclassifies pending calls after an exclusive barrier replaces their tool', async () => {
  133. const adapter = new MockAdapter([
  134. multiCall([
  135. { id: 'c1', name: 'replace', args: { id: '0' } },
  136. { id: 'c2', name: 'x', args: { id: '1' } },
  137. { id: 'c3', name: 'x', args: { id: '2' } },
  138. ]),
  139. textResponse('done'),
  140. ])
  141. const ctx = await harness(adapter)
  142. const replacement = gatedExclusiveTool('x')
  143. const disposeSafe = ctx.tools.register(defineContentToolFixture({
  144. name: 'x',
  145. description: 'initially safe',
  146. parameters: { id: { type: 'string', required: true } },
  147. isConcurrencySafe: () => true,
  148. async execute(args) { return [{ type: 'text', text: `old-${args.id}` }] },
  149. }))
  150. ctx.tools.register(defineContentToolFixture({
  151. name: 'replace',
  152. description: 'replace x',
  153. parameters: { id: { type: 'string', required: true } },
  154. async execute() {
  155. disposeSafe()
  156. ctx.tools.register(replacement.tool)
  157. return [{ type: 'text', text: 'replaced' }]
  158. },
  159. }))
  160. const agent = await ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
  161. agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
  162. await until(() => replacement.started.length === 1)
  163. await new Promise(r => setTimeout(r, 5))
  164. expect(replacement.started).toEqual(['1'])
  165. replacement.release('1')
  166. await until(() => replacement.started.length === 2)
  167. expect(replacement.started).toEqual(['1', '2'])
  168. replacement.release('2')
  169. await waitForIdle(ctx, agent)
  170. })
  171. it('stops replenishing when a result observer makes the next call exclusive', async () => {
  172. const adapter = new MockAdapter([
  173. multiCall([
  174. { id: 'c1', name: 'x', args: { id: '1' } },
  175. { id: 'c2', name: 'x', args: { id: '2' } },
  176. { id: 'c3', name: 'x', args: { id: '3' } },
  177. ]),
  178. textResponse('done'),
  179. ])
  180. const ctx = await harness(adapter, 2)
  181. const initial = gatedParallelTool('x')
  182. const replacement = gatedExclusiveTool('x')
  183. const disposeInitial = ctx.tools.register(initial.tool)
  184. ctx.on('tools/result', (exec) => {
  185. if (exec.callId !== ToolCallId('c1')) return
  186. disposeInitial()
  187. ctx.tools.register(replacement.tool)
  188. })
  189. const agent = await ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
  190. agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
  191. await until(() => initial.started.length === 2)
  192. initial.release('1')
  193. await until(() => events(agent).some(event =>
  194. event.type === 'tool/result' && event.data.message.source.callId === ToolCallId('c1')))
  195. await new Promise(r => setTimeout(r, 5))
  196. expect(replacement.started).toEqual([])
  197. initial.release('2')
  198. await until(() => replacement.started.length === 1)
  199. expect(replacement.started).toEqual(['3'])
  200. replacement.release('3')
  201. await waitForIdle(ctx, agent)
  202. })
  203. })
  204. describe('tool-call scheduler: model-order results despite out-of-order settlement', () => {
  205. it('commits tool/result in model order even when a later call settles first', async () => {
  206. const adapter = new MockAdapter([
  207. multiCall([{ id: 'c1', name: 'p', args: { id: '1' } }, { id: 'c2', name: 'p', args: { id: '2' } }]),
  208. textResponse('done'),
  209. ])
  210. const ctx = await harness(adapter)
  211. const gated = gatedParallelTool('p')
  212. ctx.tools.register(gated.tool)
  213. const agent = await ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
  214. agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
  215. await until(() => gated.started.length === 2)
  216. gated.release('2')
  217. await new Promise(r => setTimeout(r, 5))
  218. const beforeFirst = events(agent).filter(e => e.type === 'tool/result')
  219. expect(beforeFirst).toEqual([])
  220. gated.release('1')
  221. await waitForIdle(ctx, agent)
  222. const results = events(agent).filter(e => e.type === 'tool/result')
  223. expect(results.map(e => e.data.message.source.callId)).toEqual([ToolCallId('c1'), ToolCallId('c2')])
  224. })
  225. it('derived history pairs calls in model order regardless of tool/call log interleaving', async () => {
  226. const adapter = new MockAdapter([
  227. multiCall([{ id: 'c1', name: 'p', args: { id: '1' } }, { id: 'c2', name: 'p', args: { id: '2' } }]),
  228. textResponse('done'),
  229. ])
  230. const ctx = await harness(adapter)
  231. const gated = gatedParallelTool('p')
  232. ctx.tools.register(gated.tool)
  233. const agent = await ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
  234. agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
  235. await until(() => gated.started.length === 2)
  236. gated.release('2'); gated.release('1')
  237. await waitForIdle(ctx, agent)
  238. const messages = agent.session.deriveMessages()
  239. const toolResults = messages.flatMap(m => m.content.filter(b => b.type === 'tool-result'))
  240. expect(toolResults.map(b => b.toolCallId)).toEqual([ToolCallId('c1'), ToolCallId('c2')])
  241. })
  242. })
  243. describe('tool-call scheduler: rolling pool honors maxParallelToolCalls', () => {
  244. it('rejects invalid global maxParallelToolCalls config at plugin load', async () => {
  245. await expect(harness(new MockAdapter([]), 0)).rejects.toThrow()
  246. await expect(harness(new MockAdapter([]), 1.5)).rejects.toThrow()
  247. })
  248. it('defensively rejects invalid caps when direct construction bypasses the config schema', () => {
  249. // Validation precedes the turnBoundary registration, so a rejected
  250. // constructor registers nothing and needs no fiber cleanup.
  251. expect(() => new AgentLoop(new Context(), { agents: [], maxParallelToolCalls: 0 }))
  252. .toThrow('maxParallelToolCalls must be a positive integer')
  253. expect(() => new AgentLoop(new Context(), { agents: [], maxParallelToolCalls: 1.5 }))
  254. .toThrow('maxParallelToolCalls must be a positive integer')
  255. })
  256. it('defaults the cap when direct construction bypasses the config schema', async () => {
  257. const ctx = new Context()
  258. await ctx.plugin(LlmRuntime)
  259. await ctx.plugin(SessionStore)
  260. await ctx.plugin(SessionProjectionRegistry)
  261. await ctx.plugin(SystemPrompt, { personaPrefix: '' })
  262. await ctx.plugin(ToolRuntime)
  263. await ctx.plugin(AgentRegistry)
  264. const loop = new AgentLoop(ctx, { agents: [] })
  265. expect(loop.config.maxParallelToolCalls).toBe(DEFAULT_MAX_PARALLEL_TOOL_CALLS)
  266. await ctx.fiber.dispose()
  267. })
  268. it('starts at most the cap, replenishing as calls settle', async () => {
  269. const adapter = new MockAdapter([
  270. multiCall([1, 2, 3, 4].map(n => ({ id: `c${n}`, name: 'p', args: { id: String(n) } }))),
  271. textResponse('done'),
  272. ])
  273. const ctx = await harness(adapter, 2)
  274. const gated = gatedParallelTool('p')
  275. ctx.tools.register(gated.tool)
  276. const agent = await ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
  277. agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
  278. await until(() => gated.started.length === 2)
  279. await new Promise(r => setTimeout(r, 5))
  280. expect(gated.started).toEqual(['1', '2'])
  281. gated.release('1')
  282. await until(() => gated.started.length === 3)
  283. expect(gated.started).toEqual(['1', '2', '3'])
  284. expect(events(agent)
  285. .filter(e => e.type === 'tool/call' || e.type === 'tool/result')
  286. .map(e => e.type === 'tool/call'
  287. ? `${e.type}:${String(e.data.callId)}`
  288. : `${e.type}:${String(e.data.message.source.callId)}`)
  289. .slice(0, 4))
  290. .toEqual(['tool/call:c1', 'tool/call:c2', 'tool/result:c1', 'tool/call:c3'])
  291. gated.release('2'); gated.release('3')
  292. await until(() => gated.started.length === 4)
  293. gated.release('4')
  294. await waitForIdle(ctx, agent)
  295. expect(events(agent).filter(e => e.type === 'tool/result').map(e => e.data.message.source.callId))
  296. .toEqual([ToolCallId('c1'), ToolCallId('c2'), ToolCallId('c3'), ToolCallId('c4')])
  297. })
  298. it('maxParallelToolCalls: 1 is fully serial (no second start before the first settles)', async () => {
  299. const adapter = new MockAdapter([
  300. multiCall([{ id: 'c1', name: 'p', args: { id: '1' } }, { id: 'c2', name: 'p', args: { id: '2' } }]),
  301. textResponse('done'),
  302. ])
  303. const ctx = await harness(adapter, 1)
  304. const gated = gatedParallelTool('p')
  305. ctx.tools.register(gated.tool)
  306. const agent = await ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
  307. agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
  308. await until(() => gated.started.length === 1)
  309. await new Promise(r => setTimeout(r, 5))
  310. expect(gated.started).toEqual(['1'])
  311. gated.release('1')
  312. await until(() => gated.started.length === 2)
  313. gated.release('2')
  314. await waitForIdle(ctx, agent)
  315. })
  316. it('applies the configured cap to every factory-created agent', async () => {
  317. const adapter = new MockAdapter([
  318. multiCall([{ id: 'c1', name: 'p', args: { id: '1' } }, { id: 'c2', name: 'p', args: { id: '2' } }]),
  319. textResponse('done'),
  320. ])
  321. const ctx = new Context()
  322. await ctx.plugin(LlmRuntime)
  323. await ctx.plugin(SessionStore)
  324. await ctx.plugin(SessionProjectionRegistry)
  325. await ctx.plugin(SystemPrompt, { personaPrefix: '' })
  326. await ctx.plugin(ToolRuntime)
  327. await ctx.plugin(AgentRegistry)
  328. await ctx.plugin(AgentLoop, { agents: [], maxParallelToolCalls: 1 })
  329. ctx.llm.registerAdapter(['mock'], adapter)
  330. const gated = gatedParallelTool('p')
  331. ctx.tools.register(gated.tool)
  332. const agent = await ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
  333. agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
  334. await until(() => gated.started.length === 1)
  335. await new Promise(r => setTimeout(r, 5))
  336. expect(gated.started).toEqual(['1'])
  337. gated.release('1')
  338. await until(() => gated.started.length === 2)
  339. gated.release('2')
  340. await waitForIdle(ctx, agent)
  341. })
  342. })
  343. describe('tool-call scheduler: ordered middleware and additional contexts', () => {
  344. it('tools/pre-execute and tools/post-execute observe model call order', async () => {
  345. const adapter = new MockAdapter([
  346. multiCall([{ id: 'c1', name: 'p', args: { id: '1' } }, { id: 'c2', name: 'p', args: { id: '2' } }, { id: 'c3', name: 'p', args: { id: '3' } }]),
  347. textResponse('done'),
  348. ])
  349. const ctx = await harness(adapter)
  350. const gated = gatedParallelTool('p')
  351. ctx.tools.register(gated.tool)
  352. const pre: string[] = []
  353. const post: string[] = []
  354. ctx.on('tools/pre-execute', async (exec, next): Promise<PreToolDecision> => { pre.push(String(exec.callId)); return next() })
  355. ctx.on('tools/post-execute', async (exec, _result, next): Promise<PostToolDecision> => { post.push(String(exec.callId)); return next() })
  356. const agent = await ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
  357. agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
  358. await until(() => gated.started.length === 3)
  359. gated.release('3'); gated.release('2'); gated.release('1')
  360. await waitForIdle(ctx, agent)
  361. expect(pre).toEqual([ToolCallId('c1'), ToolCallId('c2'), ToolCallId('c3')].map(String))
  362. expect(post).toEqual([ToolCallId('c1'), ToolCallId('c2'), ToolCallId('c3')].map(String))
  363. })
  364. it('injects additional contexts in model call order, not settlement order', async () => {
  365. const adapter = new MockAdapter([
  366. multiCall([{ id: 'c1', name: 'p', args: { id: '1' } }, { id: 'c2', name: 'p', args: { id: '2' } }]),
  367. textResponse('done'),
  368. ])
  369. const ctx = await harness(adapter, 2)
  370. const gated = gatedParallelTool('p')
  371. ctx.tools.register(gated.tool)
  372. ctx.on('tools/post-execute', async (exec, _result): Promise<PostToolDecision> =>
  373. ({ kind: 'accept', additionalContexts: [createUserMessage({
  374. content: [{ type: 'text', text: `ctx-${exec.callId}` }], source: { kind: 'plugin', plugin: 'p' },
  375. })] }))
  376. const agent = await ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
  377. agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
  378. await until(() => gated.started.length === 2)
  379. gated.release('2'); gated.release('1')
  380. await waitForIdle(ctx, agent)
  381. const log = events(agent)
  382. const contextTexts = log.filter(e => e.type === 'user/message' && e.data.source.kind === 'plugin')
  383. .map(e => ((e.data as { content: { text: string }[] }).content[0]!).text)
  384. expect(contextTexts).toEqual(['ctx-c1', 'ctx-c2'])
  385. const lastResult = log.findLastIndex(e => e.type === 'tool/result')
  386. const firstContext = log.findIndex(e => e.type === 'user/message' && e.data.source.kind === 'plugin')
  387. expect(lastResult).toBeLessThan(firstContext)
  388. })
  389. it('orders pre-execute denials and errors without dispatching them', async () => {
  390. const adapter = new MockAdapter([
  391. multiCall([
  392. { id: 'c1', name: 'p', args: { id: '1' } },
  393. { id: 'c2', name: 'p', args: { id: '2' } },
  394. { id: 'c3', name: 'p', args: { id: '3' } },
  395. ]),
  396. textResponse('done'),
  397. ])
  398. const ctx = await harness(adapter)
  399. const gated = gatedParallelTool('p')
  400. ctx.tools.register(gated.tool)
  401. const post: string[] = []
  402. ctx.on('tools/pre-execute', async (exec, next): Promise<PreToolDecision> => {
  403. if (exec.callId === ToolCallId('c2')) return { kind: 'deny', reason: 'blocked by policy' }
  404. if (exec.callId === ToolCallId('c3')) throw new Error('pre exploded')
  405. return next()
  406. })
  407. ctx.on('tools/post-execute', async (exec, _result, next): Promise<PostToolDecision> => {
  408. post.push(String(exec.callId))
  409. return next()
  410. })
  411. const agent = await ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
  412. agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
  413. await until(() => gated.started.length === 1)
  414. gated.release('1')
  415. await waitForIdle(ctx, agent)
  416. expect(gated.started).toEqual(['1'])
  417. expect(post).toEqual(['c1', 'c2'])
  418. const results = events(agent).filter(e => e.type === 'tool/result')
  419. expect(results.map(e => e.data.message.source.callId)).toEqual([ToolCallId('c1'), ToolCallId('c2'), ToolCallId('c3')])
  420. expect((results[1]!.data.message.content[0].content[0] as { text: string }).text).toContain('blocked by policy')
  421. expect((results[2]!.data.message.content[0].content[0] as { text: string }).text).toContain('pre exploded')
  422. })
  423. })
  424. describe('tool-call scheduler: abort handling', () => {
  425. it('starts no calls when the signal is already aborted before a parallel group', async () => {
  426. const adapter = new MockAdapter([
  427. multiCall([{ id: 'c1', name: 'p', args: { id: '1' } }, { id: 'c2', name: 'p', args: { id: '2' } }]),
  428. textResponse('should never be requested'),
  429. ])
  430. const ctx = await harness(adapter)
  431. const gated = gatedParallelTool('p')
  432. ctx.tools.register(gated.tool)
  433. const agent = await ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
  434. ctx.on('session/event', (session, event) => {
  435. if (session === agent.session && event.type === 'assistant/message') {
  436. agent.cancel({ kind: 'user' })
  437. }
  438. })
  439. agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
  440. await waitForIdle(ctx, agent)
  441. expect(gated.started).toEqual([])
  442. expect(events(agent).filter(e => e.type === 'tool/call').map(e => e.data.callId))
  443. .toEqual([ToolCallId('c1'), ToolCallId('c2')])
  444. expect(events(agent).filter(e => e.type === 'tool/result').map(e => ({
  445. callId: e.data.message.source.callId,
  446. isError: e.data.message.content[0].isError,
  447. error: e.data.error,
  448. }))).toEqual([
  449. { callId: ToolCallId('c1'), isError: true, error: { name: 'AbortError', code: TOOL_ABORTED_BEFORE_DISPATCH } },
  450. { callId: ToolCallId('c2'), isError: true, error: { name: 'AbortError', code: TOOL_ABORTED_BEFORE_DISPATCH } },
  451. ])
  452. })
  453. it('skips dispatch and stops starting siblings when abort fires during ordered pre-execute', async () => {
  454. const adapter = new MockAdapter([
  455. multiCall([{ id: 'c1', name: 'p', args: { id: '1' } }, { id: 'c2', name: 'p', args: { id: '2' } }]),
  456. textResponse('should never be requested'),
  457. ])
  458. const ctx = await harness(adapter)
  459. const gated = gatedParallelTool('p')
  460. ctx.tools.register(gated.tool)
  461. const agent = await ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
  462. ctx.on('tools/pre-execute', async (exec, next): Promise<PreToolDecision> => {
  463. if (exec.callId === ToolCallId('c1')) {
  464. agent.cancel({ kind: 'user' })
  465. }
  466. return next()
  467. })
  468. agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
  469. await waitForIdle(ctx, agent)
  470. expect(gated.started).toEqual([])
  471. expect(events(agent).filter(e => e.type === 'tool/call').map(e => e.data.callId))
  472. .toEqual([ToolCallId('c1'), ToolCallId('c2')])
  473. expect(events(agent).filter(e => e.type === 'tool/result').map(e => ({
  474. callId: e.data.message.source.callId,
  475. isError: e.data.message.content[0].isError,
  476. error: e.data.error,
  477. }))).toEqual([
  478. { callId: ToolCallId('c1'), isError: true, error: { name: 'AbortError', code: TOOL_ABORTED_BEFORE_DISPATCH } },
  479. { callId: ToolCallId('c2'), isError: true, error: { name: 'AbortError', code: TOOL_ABORTED_BEFORE_DISPATCH } },
  480. ])
  481. })
  482. it('stops replenishing after abort, commits started results, and parks accepted additional contexts', async () => {
  483. const adapter = new MockAdapter([
  484. multiCall([1, 2, 3, 4].map(n => ({ id: `c${n}`, name: 'p', args: { id: String(n) } }))),
  485. textResponse('after wake'),
  486. ])
  487. const ctx = await harness(adapter, 2)
  488. const gated = gatedParallelTool('p')
  489. ctx.tools.register(gated.tool)
  490. ctx.on('tools/post-execute', async (exec, _result, next): Promise<PostToolDecision> => ({
  491. ...await next(),
  492. additionalContexts: [createUserMessage({
  493. content: [{ type: 'text', text: `ctx-${exec.callId}` }], source: { kind: 'plugin', plugin: 'p' },
  494. })],
  495. }))
  496. const agent = await ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
  497. agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
  498. await until(() => gated.started.length === 2)
  499. agent.cancel({ kind: 'user' })
  500. gated.release('1')
  501. gated.release('2')
  502. await waitForIdle(ctx, agent)
  503. expect(gated.started).toEqual(['1', '2'])
  504. expect(events(agent).filter(e => e.type === 'tool/call').map(e => e.data.callId))
  505. .toEqual([ToolCallId('c1'), ToolCallId('c2'), ToolCallId('c3'), ToolCallId('c4')])
  506. expect(events(agent).filter(e => e.type === 'tool/result').map(e => e.data.message.source.callId))
  507. .toEqual([ToolCallId('c1'), ToolCallId('c2'), ToolCallId('c3'), ToolCallId('c4')])
  508. expect(events(agent).filter(e => e.type === 'tool/result').slice(-2).map(e => ({
  509. callId: e.data.message.source.callId,
  510. isError: e.data.message.content[0].isError,
  511. error: e.data.error,
  512. })))
  513. .toEqual([
  514. {
  515. callId: ToolCallId('c3'),
  516. isError: true,
  517. error: { name: 'AbortError', code: TOOL_ABORTED_BEFORE_DISPATCH },
  518. },
  519. {
  520. callId: ToolCallId('c4'),
  521. isError: true,
  522. error: { name: 'AbortError', code: TOOL_ABORTED_BEFORE_DISPATCH },
  523. },
  524. ])
  525. const settled = events(agent).filter(e => e.type === 'tool/result'
  526. || (e.type === 'user/message' && e.data.source.kind === 'plugin'))
  527. expect(settled.map(e => e.type))
  528. .toEqual(['tool/result', 'tool/result', 'tool/result', 'tool/result'])
  529. expect(agent.inbox.nextStep.map(message => message.content[0]))
  530. .toEqual([
  531. { type: 'text', text: 'ctx-c1' },
  532. { type: 'text', text: 'ctx-c2' },
  533. ])
  534. const idle = waitForIdle(ctx, agent)
  535. agent.followup(createUserMessage({ content: [{ type: 'text', text: 'wake' }], source: { kind: 'user' } }))
  536. await idle
  537. expect(events(agent).flatMap(e =>
  538. e.type === 'user/message'
  539. && e.data.source.kind === 'plugin'
  540. && e.data.content[0]?.type === 'text'
  541. ? [e.data.content[0].text]
  542. : []))
  543. .toEqual(['ctx-c1', 'ctx-c2'])
  544. })
  545. it('does not run an exclusive barrier after a parallel group aborts', async () => {
  546. const adapter = new MockAdapter([
  547. multiCall([
  548. { id: 'c1', name: 'p', args: { id: '1' } },
  549. { id: 'c2', name: 'p', args: { id: '2' } },
  550. { id: 'c3', name: 'x', args: { id: '3' } },
  551. ]),
  552. textResponse('should never be requested'),
  553. ])
  554. const ctx = await harness(adapter, 2)
  555. const gated = gatedParallelTool('p')
  556. const exclusive: string[] = []
  557. ctx.tools.register(gated.tool)
  558. ctx.tools.register(defineContentToolFixture({
  559. name: 'x',
  560. description: 'exclusive',
  561. parameters: { id: { type: 'string', required: true } },
  562. async execute(args) { exclusive.push(args.id); return [{ type: 'text', text: 'x' }] },
  563. }))
  564. const agent = await ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
  565. agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
  566. await until(() => gated.started.length === 2)
  567. agent.cancel({ kind: 'user' })
  568. gated.release('1')
  569. gated.release('2')
  570. await waitForIdle(ctx, agent)
  571. expect(exclusive).toEqual([])
  572. expect(events(agent).filter(e => e.type === 'tool/call').map(e => e.data.callId))
  573. .toEqual([ToolCallId('c1'), ToolCallId('c2'), ToolCallId('c3')])
  574. expect(events(agent).filter(e => e.type === 'tool/result').at(-1)?.data)
  575. .toMatchObject({
  576. message: {
  577. source: { kind: 'tool', callId: ToolCallId('c3') },
  578. content: [{ isError: true }],
  579. },
  580. error: { name: 'AbortError', code: TOOL_ABORTED_BEFORE_DISPATCH },
  581. })
  582. })
  583. })
  584. describe('tool-call scheduler: failure quiescence', () => {
  585. it('stops new dispatches and drains started bodies before surfacing the first failure', async () => {
  586. const adapter = new MockAdapter([
  587. multiCall([
  588. { id: 'c1', name: 'p', args: { id: '1' } },
  589. { id: 'c2', name: 'p', args: { id: '2' } },
  590. { id: 'c3', name: 'p', args: { id: '3' } },
  591. ]),
  592. ])
  593. const ctx = await harness(adapter, 3)
  594. const gated = gatedParallelTool('p')
  595. ctx.tools.register(gated.tool)
  596. // The registry contains expected failures as results; replace its internal
  597. // view only to inject the invariant violation this boundary must contain.
  598. const scheduler = ctx.tools[TOOL_RUNTIME_SCHEDULER]
  599. const prepare = scheduler.prepare.bind(scheduler)
  600. const dispatch = scheduler.dispatch.bind(scheduler)
  601. const prepareGate = Promise.withResolvers<undefined>()
  602. let thirdPrepareEntered = false
  603. scheduler.prepare = async (exec) => {
  604. const prepared = await prepare(exec)
  605. if (exec.callId === ToolCallId('c3')) {
  606. thirdPrepareEntered = true
  607. await prepareGate.promise
  608. }
  609. return prepared
  610. }
  611. const schedulerError = new Error('scheduler exploded')
  612. const drainedError = new Error('sibling failed while draining')
  613. let rejectFirst: ((error: Error) => void) | undefined
  614. scheduler.dispatch = exec => exec.callId === ToolCallId('c1')
  615. ? new Promise((_resolve, reject) => { rejectFirst = reject })
  616. : dispatch(exec).then(() => { throw drainedError })
  617. const agent = await ctx.agentLoop.create(SessionId('scheduler-failure'), { provider: 'mock', model: 'mock' })
  618. let idle = false
  619. const idlePromise = waitForIdle(ctx, agent).then(() => { idle = true })
  620. agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
  621. await until(() => gated.started.includes('2') && thirdPrepareEntered && rejectFirst !== undefined)
  622. rejectFirst?.(schedulerError)
  623. await new Promise<void>(resolve => setImmediate(resolve))
  624. prepareGate.resolve(undefined)
  625. await new Promise<void>(resolve => setImmediate(resolve))
  626. const startedBeforeDrain = [...gated.started]
  627. const idleBeforeDrain = idle
  628. const turnEndBeforeDrain = events(agent).find(event => event.type === 'turn/end')
  629. for (const id of gated.pending()) gated.release(id)
  630. await idlePromise
  631. expect(startedBeforeDrain).toEqual(['2'])
  632. expect(idleBeforeDrain).toBe(false)
  633. expect(turnEndBeforeDrain).toBeUndefined()
  634. expect(gated.pending()).toEqual([])
  635. expect(events(agent).findLast(event => event.type === 'turn/end')).toMatchObject({
  636. data: { reason: { kind: 'error', error: { message: schedulerError.message, code: 'UNKNOWN' } } },
  637. })
  638. })
  639. })
  640. describe('PTC mode native-tool denial through the agent loop', () => {
  641. /** A minimal in-process code runtime for test purposes — never actually runs. */
  642. class FakeCodeRuntime extends CodeRuntime {
  643. readonly language = 'typescript'
  644. readonly isolation = 'fake' as const
  645. async run(_request: CodeRunRequest): Promise<CodeRunResult> {
  646. return { logs: [] }
  647. }
  648. }
  649. async function ptcModeHarness(adapter: MockAdapter) {
  650. const ctx = new Context()
  651. await ctx.plugin(LlmRuntime)
  652. await ctx.plugin(SessionStore)
  653. await ctx.plugin(SessionProjectionRegistry)
  654. await ctx.plugin(SystemPrompt, { personaPrefix: '' })
  655. await ctx.plugin(ToolRuntime, { mode: 'ptc' })
  656. // eslint-disable-next-line @typescript-eslint/no-explicit-any -- FakeCodeRuntime is an internal test helper with an opaque type shape
  657. await ctx.plugin(FakeCodeRuntime as any)
  658. await ctx.plugin(AgentRegistry)
  659. await ctx.plugin(AgentLoop, { agents: [] })
  660. ctx.llm.registerAdapter(['mock'], adapter)
  661. return ctx
  662. }
  663. it('denies a model-direct native-tool call under PTC mode: tool body never runs and session records UNKNOWN_TOOL', async () => {
  664. let toolInvoked = false
  665. const tool = defineContentToolFixture({
  666. name: 'write',
  667. description: 'Write a file.',
  668. parameters: {
  669. file_path: { type: 'string', required: true },
  670. content: { type: 'string', required: true },
  671. },
  672. async execute(_args, _exec) {
  673. toolInvoked = true
  674. return [{ type: 'text', text: 'written' }]
  675. },
  676. })
  677. // Scripted model emits a native tool call under PTC mode — the wire
  678. // never advertised it, but a non-compliant provider may still emit one.
  679. const adapter = new MockAdapter([
  680. [
  681. ...multiCall([{ id: 'call-1', name: 'write', args: { file_path: '/tmp/test', content: 'hello' } }]),
  682. ...textResponse('ok'),
  683. ],
  684. ])
  685. const ctx = await ptcModeHarness(adapter)
  686. ctx.tools.register(tool)
  687. const agent = await ctx.agentLoop.create(SessionId('code-native'), { provider: 'mock', model: 'mock' })
  688. agent.followup(createUserMessage({ content: [{ type: 'text', text: 'write a file' }], source: { kind: 'user' } }))
  689. await waitForIdle(ctx, agent)
  690. // The tool body must NOT have executed — the collapse denied the call
  691. // at createExecution, before the body could start.
  692. expect(toolInvoked).toBe(false)
  693. // The session must record a tool/result with UNKNOWN_TOOL error so the
  694. // transcript faithfully captures that the call was denied.
  695. const sessionEvents = events(agent)
  696. const toolResult = sessionEvents.find(e => e.type === 'tool/result')
  697. expect(toolResult).toBeDefined()
  698. expect(toolResult!.data.error).toMatchObject({
  699. name: 'ToolNotFoundError',
  700. code: 'UNKNOWN_TOOL',
  701. })
  702. })
  703. })