tools.spec.ts 39 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042
  1. import { describe, expect, expectTypeOf, it } from 'vitest'
  2. import { Context } from 'cordis'
  3. import { CallId, HarnessError } from '@deepseek-ai/dsh-llm'
  4. import SystemPrompt from '@deepseek-ai/dsh-system-prompt'
  5. import ToolRegistry, {
  6. defineTool, schemaSpecToJsonSchema, validateArgs, ToolArgsError, ToolNotFoundError,
  7. type InferArgs, type SchemaSpec, type PreToolDecision, type PostToolDecision,
  8. } from '@deepseek-ai/dsh-tools'
  9. async function setup() {
  10. const ctx = new Context()
  11. await ctx.plugin(SystemPrompt)
  12. await ctx.plugin(ToolRegistry)
  13. return ctx
  14. }
  15. const echoTool = defineTool({
  16. name: 'echo',
  17. description: 'echo arguments back',
  18. parameters: { text: { type: 'string' } },
  19. async execute(args) {
  20. return [{ type: 'text' as const, text: args.text ?? '' }]
  21. },
  22. })
  23. describe('ToolRegistry', () => {
  24. it('registers tools, exposes schemas, and feeds the system-prompt assembly', async () => {
  25. const ctx = await setup()
  26. ctx.tools.register(echoTool)
  27. expect(ctx.tools.schemas()).toEqual([{
  28. name: 'echo',
  29. description: 'echo arguments back',
  30. parameters: { type: 'object', properties: { text: { type: 'string' } } },
  31. }])
  32. // schemas() result must not leak execute — ToolSchema deliberately has no
  33. // 'execute' key, so widen through unknown to probe for the absent property
  34. expect((ctx.tools.schemas()[0] as unknown as Record<string, unknown>).execute).toBeUndefined()
  35. const assembly = await ctx.systemPrompt.assemble()
  36. expect(assembly.tools.map(t => t.name)).toEqual(['echo'])
  37. })
  38. it('schemas() drops the UI presentation callbacks — they must never reach the model', async () => {
  39. const ctx = await setup()
  40. // A tool that declares presentCall/presentResult (functions). schemas() feeds
  41. // the system-prompt assembly → the model request, so those callbacks (and
  42. // `execute`) must be stripped: a function in the JSON tool schema would
  43. // corrupt the request. schemas() is an explicit allowlist, so it can't leak.
  44. ctx.tools.register(defineTool({
  45. name: 'present',
  46. description: 'has presenters',
  47. parameters: { x: { type: 'string', required: true } },
  48. async execute() { return [] },
  49. presentCall: args => ({ card: 'generic', title: args.x }),
  50. presentResult: (args, result) => ({ card: 'generic', title: args.x, content: result.content }),
  51. }))
  52. const schema = ctx.tools.schemas()[0] as unknown as Record<string, unknown>
  53. expect(Object.keys(schema).sort()).toEqual(['description', 'name', 'parameters'])
  54. expect(schema.presentCall).toBeUndefined()
  55. expect(schema.presentResult).toBeUndefined()
  56. expect(schema.execute).toBeUndefined()
  57. })
  58. it('executes a tool and returns its content', async () => {
  59. const ctx = await setup()
  60. ctx.tools.register(echoTool)
  61. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'echo', arguments: { text: 'hi' } })
  62. expect(result).toEqual({ callId: CallId('c1'), content: [{ type: 'text', text: 'hi' }], isError: false })
  63. })
  64. it('threads a tool-attached meta (object return form) onto the result', async () => {
  65. const ctx = await setup()
  66. ctx.tools.register({
  67. ...echoTool,
  68. name: 'meta-tool',
  69. async execute() {
  70. return { content: [{ type: 'text', text: 'ok' }], meta: { diffs: [{ path: 'a', oldText: null, newText: 'x' }] } }
  71. },
  72. })
  73. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'meta-tool', arguments: {} })
  74. expect(result).toEqual({
  75. callId: CallId('c1'),
  76. content: [{ type: 'text', text: 'ok' }],
  77. isError: false,
  78. meta: { diffs: [{ path: 'a', oldText: null, newText: 'x' }] },
  79. })
  80. })
  81. it('omits meta when the object return form supplies none', async () => {
  82. const ctx = await setup()
  83. ctx.tools.register({
  84. ...echoTool,
  85. name: 'no-meta-tool',
  86. async execute() {
  87. return { content: [{ type: 'text', text: 'ok' }] }
  88. },
  89. })
  90. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'no-meta-tool', arguments: {} })
  91. expect(result).toEqual({ callId: CallId('c1'), content: [{ type: 'text', text: 'ok' }], isError: false })
  92. expect('meta' in result).toBe(false)
  93. })
  94. it('returns isError results for unknown tools and throwing tools', async () => {
  95. const ctx = await setup()
  96. ctx.tools.register({
  97. ...echoTool,
  98. name: 'boom',
  99. async execute() {
  100. throw new Error('exploded')
  101. },
  102. })
  103. const unknown = await ctx.tools.execute({ callId: CallId('c1'), name: 'nope', arguments: {} })
  104. expect(unknown.isError).toBe(true)
  105. expect(unknown.content[0]).toMatchObject({ text: 'Error: unknown tool "nope"' })
  106. // An unknown tool is a routable failure class, same as a tool-thrown one.
  107. expect(unknown.error).toEqual({ name: 'ToolNotFoundError', code: 'UNKNOWN_TOOL' })
  108. const thrown = await ctx.tools.execute({ callId: CallId('c2'), name: 'boom', arguments: {} })
  109. expect(thrown.isError).toBe(true)
  110. expect(thrown.content[0]).toMatchObject({ text: 'Error: exploded' })
  111. })
  112. it('ToolNotFoundError carries the tool name and a stable code', async () => {
  113. const { HarnessError } = await import('@deepseek-ai/dsh-llm')
  114. const err = new ToolNotFoundError('ghost')
  115. expect(err).toBeInstanceOf(HarnessError)
  116. expect(err.name).toBe('ToolNotFoundError')
  117. expect(err.code).toBe('UNKNOWN_TOOL')
  118. expect(err.toolName).toBe('ghost')
  119. expect(err.message).toBe('unknown tool "ghost"')
  120. })
  121. it('lets a tools/pre-execute listener deny a call (permission pattern)', async () => {
  122. const ctx = await setup()
  123. ctx.tools.register(echoTool)
  124. ctx.on('tools/pre-execute', async (exec, next): Promise<PreToolDecision> => {
  125. if (exec.name === 'echo') return { kind: 'deny', reason: 'denied by policy' }
  126. return next()
  127. })
  128. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'echo', arguments: { text: 'hi' } })
  129. expect(result.isError).toBe(true)
  130. expect(result.content[0]).toMatchObject({ text: 'Error: denied by policy' })
  131. })
  132. it('an ask decision degrades to deny until the permission system lands', async () => {
  133. const ctx = await setup()
  134. ctx.tools.register(echoTool)
  135. ctx.on('tools/pre-execute', async (_exec, _next): Promise<PreToolDecision> =>
  136. ({ kind: 'ask', reason: 'needs approval' }))
  137. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'echo', arguments: { text: 'hi' } })
  138. expect(result.isError).toBe(true)
  139. expect(result.content[0]).toMatchObject({ text: 'Error: needs approval' })
  140. })
  141. it('an ask decision with no reason degrades to deny with a default message', async () => {
  142. const ctx = await setup()
  143. ctx.tools.register(echoTool)
  144. ctx.on('tools/pre-execute', async (_exec, _next): Promise<PreToolDecision> => ({ kind: 'ask' }))
  145. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'echo', arguments: { text: 'hi' } })
  146. expect(result.isError).toBe(true)
  147. expect(result.content[0]).toMatchObject({ text: 'Error: tool "echo" requires approval (not yet supported)' })
  148. })
  149. it('a tools/post-execute listener can replace the result content (accept) ', async () => {
  150. const ctx = await setup()
  151. ctx.tools.register(echoTool)
  152. ctx.on('tools/post-execute', async (_exec, _result, _next): Promise<PostToolDecision> =>
  153. ({ kind: 'accept', content: [{ type: 'text', text: 'rewritten' }] }))
  154. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'echo', arguments: { text: 'hi' } })
  155. expect(result.isError).toBe(false)
  156. expect(result.content[0]).toMatchObject({ text: 'rewritten' })
  157. })
  158. it('a tools/post-execute block turns the call into an isError with corrective feedback', async () => {
  159. const ctx = await setup()
  160. ctx.tools.register(echoTool)
  161. ctx.on('tools/post-execute', async (_exec, _result, _next): Promise<PostToolDecision> =>
  162. ({ kind: 'block', feedback: [{ type: 'text', text: 'output rejected: try again' }] }))
  163. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'echo', arguments: { text: 'hi' } })
  164. expect(result.isError).toBe(true)
  165. expect(result.content[0]).toMatchObject({ text: 'output rejected: try again' })
  166. })
  167. it('a block decision can ALSO attach additionalContext', async () => {
  168. const ctx = await setup()
  169. ctx.tools.register(echoTool)
  170. ctx.on('tools/post-execute', async (_exec, _result, _next): Promise<PostToolDecision> =>
  171. ({
  172. kind: 'block',
  173. feedback: [{ type: 'text', text: 'rejected' }],
  174. additionalContext: { content: [{ type: 'text', text: 'why it was rejected' }], source: { kind: 'plugin', plugin: 'test' } },
  175. }))
  176. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'echo', arguments: { text: 'hi' } })
  177. expect(result.isError).toBe(true)
  178. expect(result.content[0]).toMatchObject({ text: 'rejected' })
  179. expect(result.additionalContext).toMatchObject({ content: [{ text: 'why it was rejected' }], source: { kind: 'plugin', plugin: 'test' } })
  180. })
  181. it('a post-execute additionalContext rides on the result for the loop to buffer', async () => {
  182. const ctx = await setup()
  183. ctx.tools.register(echoTool)
  184. ctx.on('tools/post-execute', async (_exec, _result, _next): Promise<PostToolDecision> =>
  185. ({ kind: 'accept', additionalContext: { content: [{ type: 'text', text: 'fyi' }], source: { kind: 'plugin', plugin: 'test' } } }))
  186. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'echo', arguments: { text: 'hi' } })
  187. expect(result.additionalContext).toMatchObject({ content: [{ text: 'fyi' }], source: { kind: 'plugin', plugin: 'test' } })
  188. })
  189. it('a post-execute listener mutating the result object cannot corrupt callId/isError/error', async () => {
  190. // The decision is the ONLY sanctioned channel to change the outcome. A
  191. // listener that reaches in and mutates the passed result reference (flipping
  192. // isError, rewriting callId, attaching a bogus error) must NOT affect what
  193. // execute() returns — the registry snapshots the authoritative fields before
  194. // the waterfall and rebuilds from the snapshot + decision.
  195. const ctx = await setup()
  196. ctx.tools.register(echoTool)
  197. ctx.on('tools/post-execute', async (_exec, result, next) => {
  198. const mutable = result as { callId: string; isError: boolean; error?: unknown; content: unknown[] }
  199. mutable.callId = 'hijacked'
  200. mutable.isError = true
  201. mutable.error = { name: 'Evil', code: 'EVIL' }
  202. mutable.content.push({ type: 'text', text: 'INJECTED' }) // in-place array mutation
  203. return next() // delegate to the default accept — no decision-level override
  204. })
  205. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'echo', arguments: { text: 'hi' } })
  206. expect(result.callId).toBe(CallId('c1')) // authoritative exec.callId, not 'hijacked'
  207. expect(result.isError).toBe(false) // the real (successful) dispatch outcome
  208. expect(result.error).toBeUndefined() // no listener-injected error
  209. expect(result.content).toHaveLength(1) // the in-place push did not leak in
  210. expect(result.content[0]).toMatchObject({ text: 'hi' })
  211. expect(result.content.some(b => (b as { text?: string }).text === 'INJECTED')).toBe(false)
  212. })
  213. it('composes pre + post waterfalls around dispatch (sandbox-wrap pattern)', async () => {
  214. const ctx = await setup()
  215. ctx.tools.register(echoTool)
  216. const order: string[] = []
  217. ctx.on('tools/pre-execute', async (_exec, next) => {
  218. order.push('pre:before')
  219. const decision = await next()
  220. order.push('pre:after')
  221. return decision
  222. })
  223. ctx.on('tools/post-execute', async (_exec, _result, next) => {
  224. order.push('post:before')
  225. const decision = await next()
  226. order.push('post:after')
  227. return decision
  228. })
  229. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'echo', arguments: { text: 'x' } })
  230. expect(result.isError).toBe(false)
  231. // pre runs fully (gate) before dispatch, then post runs over the result.
  232. expect(order).toEqual(['pre:before', 'pre:after', 'post:before', 'post:after'])
  233. })
  234. it('returns an isError result when a tools/pre-execute listener throws', async () => {
  235. const ctx = await setup()
  236. ctx.tools.register(echoTool)
  237. ctx.on('tools/pre-execute', async () => {
  238. throw new Error('permission hook broke')
  239. })
  240. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'echo', arguments: { text: 'hi' } })
  241. expect(result).toEqual({
  242. callId: CallId('c1'),
  243. content: [{ type: 'text', text: 'Error: permission hook broke' }],
  244. isError: true,
  245. })
  246. })
  247. it('returns an isError result when a tools/post-execute listener throws', async () => {
  248. const ctx = await setup()
  249. ctx.tools.register(echoTool)
  250. ctx.on('tools/post-execute', async () => {
  251. throw new Error('post hook broke')
  252. })
  253. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'echo', arguments: { text: 'hi' } })
  254. expect(result).toEqual({
  255. callId: CallId('c1'),
  256. content: [{ type: 'text', text: 'Error: post hook broke' }],
  257. isError: true,
  258. })
  259. })
  260. it('preserves structured error info when a tools/pre-execute listener throws HarnessError', async () => {
  261. const ctx = await setup()
  262. ctx.tools.register(echoTool)
  263. ctx.on('tools/pre-execute', async () => {
  264. throw new HarnessError('denied', 'DENIED')
  265. })
  266. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'echo', arguments: { text: 'hi' } })
  267. expect(result).toMatchObject({
  268. callId: CallId('c1'),
  269. isError: true,
  270. error: { name: 'HarnessError', code: 'DENIED' },
  271. })
  272. })
  273. it('schemas() snapshots tool schemas instead of exposing registry objects', async () => {
  274. const ctx = await setup()
  275. ctx.tools.register(echoTool)
  276. const first = ctx.tools.schemas()
  277. const firstParameters = first[0]!.parameters as { properties: Record<string, unknown> }
  278. firstParameters.properties['mutated'] = { type: 'string' }
  279. first[0]!.description = 'mutated'
  280. expect(ctx.tools.schemas()).toEqual([{
  281. name: 'echo',
  282. description: 'echo arguments back',
  283. parameters: { type: 'object', properties: { text: { type: 'string' } } },
  284. }])
  285. })
  286. it('rejects duplicate names and unregisters on fiber dispose (HMR safety)', async () => {
  287. const ctx = await setup()
  288. ctx.tools.register(echoTool)
  289. expect(() => ctx.tools.register(echoTool)).toThrow('already registered')
  290. const fiber = await ctx.plugin(Object.assign((inner: Context) => {
  291. inner.tools.register({ ...echoTool, name: 'scoped' })
  292. }, { inject: ['tools'] }))
  293. expect(ctx.tools.schemas().map(t => t.name)).toEqual(['echo', 'scoped'])
  294. await fiber.dispose()
  295. expect(ctx.tools.schemas().map(t => t.name)).toEqual(['echo'])
  296. })
  297. it('returns a callable disposer from register() that unregisters the tool', async () => {
  298. const ctx = await setup()
  299. ctx.tools.register(echoTool)
  300. // Register a second tool and call its returned disposer directly
  301. const dispose = ctx.tools.register({ ...echoTool, name: 'disposable' })
  302. expect(ctx.tools.schemas().map(t => t.name)).toEqual(['echo', 'disposable'])
  303. dispose()
  304. expect(ctx.tools.schemas().map(t => t.name)).toEqual(['echo'])
  305. })
  306. it('rolls back the tool entry when a tools/change listener throws (P1-1)', async () => {
  307. const ctx = await setup()
  308. let threw = false
  309. ctx.on('tools/change', () => {
  310. if (!threw) { threw = true; throw new Error('boom change listener') }
  311. })
  312. // The throwing emit must roll the entry back, not leak it.
  313. expect(() => ctx.tools.register(echoTool)).toThrow('boom change listener')
  314. expect(ctx.tools.get('echo')).toBeUndefined() // rolled back, not leaked
  315. expect(ctx.tools.schemas()).toHaveLength(0)
  316. // A subsequent listener-free register of the SAME name succeeds and is
  317. // exposed exactly once (the duplicate-name check is not wedged).
  318. const dispose = ctx.tools.register(echoTool)
  319. expect(ctx.tools.schemas().map(t => t.name)).toEqual(['echo'])
  320. dispose()
  321. expect(ctx.tools.get('echo')).toBeUndefined()
  322. })
  323. })
  324. describe('defineTool / schema DSL', () => {
  325. it('converts SchemaSpec to standard JSON Schema with required array', () => {
  326. const spec = {
  327. path: { type: 'string', required: true, description: 'Absolute path' },
  328. offset: { type: 'number' },
  329. limit: { type: 'number', description: 'Max lines' },
  330. } satisfies SchemaSpec
  331. const jsonSchema = schemaSpecToJsonSchema(spec)
  332. expect(jsonSchema).toEqual({
  333. type: 'object',
  334. properties: {
  335. path: { type: 'string', description: 'Absolute path' },
  336. offset: { type: 'number' },
  337. limit: { type: 'number', description: 'Max lines' },
  338. },
  339. required: ['path'],
  340. })
  341. })
  342. it('handles empty spec (no properties, no required)', () => {
  343. expect(schemaSpecToJsonSchema({})).toEqual({
  344. type: 'object',
  345. properties: {},
  346. })
  347. })
  348. it('handles nested object spec', () => {
  349. const spec = {
  350. config: {
  351. type: 'object',
  352. required: true,
  353. properties: {
  354. host: { type: 'string', required: true },
  355. port: { type: 'number' },
  356. },
  357. },
  358. } satisfies SchemaSpec
  359. const jsonSchema = schemaSpecToJsonSchema(spec)
  360. expect(jsonSchema).toEqual({
  361. type: 'object',
  362. properties: {
  363. config: {
  364. type: 'object',
  365. properties: {
  366. host: { type: 'string' },
  367. port: { type: 'number' },
  368. },
  369. required: ['host'],
  370. },
  371. },
  372. required: ['config'],
  373. })
  374. })
  375. it('defineTool returns a valid ToolDefinition with typed execute', async () => {
  376. const ctx = await setup()
  377. const tool = defineTool({
  378. name: 'typed-echo',
  379. description: 'A typed echo tool',
  380. parameters: {
  381. text: { type: 'string', required: true },
  382. uppercase: { type: 'boolean' },
  383. },
  384. async execute(args) {
  385. // args is typed: { text: string; uppercase?: boolean }
  386. const result = args.uppercase ? args.text.toUpperCase() : args.text
  387. return [{ type: 'text', text: result }]
  388. },
  389. })
  390. ctx.tools.register(tool)
  391. expect(ctx.tools.schemas()).toEqual([{
  392. name: 'typed-echo',
  393. description: 'A typed echo tool',
  394. parameters: {
  395. type: 'object',
  396. properties: {
  397. text: { type: 'string' },
  398. uppercase: { type: 'boolean' },
  399. },
  400. required: ['text'],
  401. },
  402. }])
  403. const result = await ctx.tools.execute({
  404. callId: CallId('c1'),
  405. name: 'typed-echo',
  406. arguments: { text: 'hello', uppercase: true },
  407. })
  408. expect(result.isError).toBe(false)
  409. expect(result.content).toEqual([{ type: 'text', text: 'HELLO' }])
  410. })
  411. it('type-level: InferArgs maps required properties to non-optional', () => {
  412. // Compile-time check: if this compiles, InferArgs is correct.
  413. // args.a is string (required), args.b is number|undefined (optional).
  414. const tool = defineTool({
  415. name: 'type-check',
  416. description: '',
  417. parameters: { a: { type: 'string' as const, required: true as const }, b: { type: 'number' as const } },
  418. async execute(args) {
  419. // Verify types at runtime via typeof
  420. expect(typeof args.a).toBe('string')
  421. // args.b should be undefined when not provided
  422. void args
  423. return [{ type: 'text', text: args.a }]
  424. },
  425. })
  426. void tool
  427. })
  428. it('registry round-trips a defineTool definition (register→schemas→execute)', async () => {
  429. const ctx = await setup()
  430. ctx.tools.register(defineTool({
  431. name: 'roundtrip',
  432. description: 'Round-trip test',
  433. parameters: {
  434. req: { type: 'string', required: true },
  435. opt: { type: 'number', description: 'Optional number' },
  436. },
  437. async execute(args) {
  438. return [{ type: 'text', text: `${args.req}:${args.opt ?? 'none'}` }]
  439. },
  440. }))
  441. // Schema round-trip: schemas() returns standard JSON Schema
  442. const schemas = ctx.tools.schemas()
  443. expect(schemas).toHaveLength(1)
  444. expect(schemas[0]!.parameters).toEqual({
  445. type: 'object',
  446. properties: {
  447. req: { type: 'string' },
  448. opt: { type: 'number', description: 'Optional number' },
  449. },
  450. required: ['req'],
  451. })
  452. // Execution round-trip
  453. const result = await ctx.tools.execute({
  454. callId: CallId('c1'),
  455. name: 'roundtrip',
  456. arguments: { req: 'hello' },
  457. })
  458. expect(result.isError).toBe(false)
  459. expect(result.content).toEqual([{ type: 'text', text: 'hello:none' }])
  460. })
  461. it('still accepts raw JSON-Schema ToolDefinition directly (MCP interop)', async () => {
  462. const ctx = await setup()
  463. ctx.tools.register({
  464. name: 'raw-tool',
  465. description: 'Raw JSON Schema tool (like an MCP adapter would register)',
  466. parameters: {
  467. type: 'object',
  468. properties: { path: { type: 'string' } },
  469. required: ['path'],
  470. },
  471. async execute(args: unknown) {
  472. const p = args as { path: string }
  473. return [{ type: 'text', text: p.path }]
  474. },
  475. })
  476. const schemas = ctx.tools.schemas()
  477. expect(schemas[0]!.parameters).toEqual({
  478. type: 'object',
  479. properties: { path: { type: 'string' } },
  480. required: ['path'],
  481. })
  482. const result = await ctx.tools.execute({
  483. callId: CallId('c1'),
  484. name: 'raw-tool',
  485. arguments: { path: '/tmp' },
  486. })
  487. expect(result.isError).toBe(false)
  488. expect(result.content).toEqual([{ type: 'text', text: '/tmp' }])
  489. })
  490. })
  491. describe('schema DSL edge cases', () => {
  492. it('emits enum values in JSON Schema property', () => {
  493. const spec = {
  494. color: { type: 'string', enum: ['red', 'green', 'blue'], description: 'Color choice' },
  495. } satisfies SchemaSpec
  496. const jsonSchema = schemaSpecToJsonSchema(spec)
  497. expect(jsonSchema.properties['color']).toMatchObject({
  498. type: 'string',
  499. enum: ['red', 'green', 'blue'],
  500. description: 'Color choice',
  501. })
  502. })
  503. it('emits default value in JSON Schema property', () => {
  504. const spec = {
  505. limit: { type: 'number', default: 25 },
  506. } satisfies SchemaSpec
  507. const jsonSchema = schemaSpecToJsonSchema(spec)
  508. expect(jsonSchema.properties['limit']).toMatchObject({
  509. type: 'number',
  510. default: 25,
  511. })
  512. })
  513. it('handles array items without nested properties (plain type array)', () => {
  514. const spec = {
  515. tags: { type: 'array', items: { type: 'string' } },
  516. } satisfies SchemaSpec
  517. const jsonSchema = schemaSpecToJsonSchema(spec)
  518. expect(jsonSchema.properties['tags']).toEqual({
  519. type: 'array',
  520. items: { type: 'string' },
  521. })
  522. })
  523. it('handles enum and default together in one property', () => {
  524. const spec = {
  525. level: { type: 'string', enum: ['low', 'high'], default: 'low' },
  526. } satisfies SchemaSpec
  527. const jsonSchema = schemaSpecToJsonSchema(spec)
  528. expect(jsonSchema.properties['level']).toMatchObject({
  529. type: 'string',
  530. enum: ['low', 'high'],
  531. default: 'low',
  532. })
  533. })
  534. it('omits description, enum, default keys when not specified', () => {
  535. const spec = {
  536. bare: { type: 'string' },
  537. } satisfies SchemaSpec
  538. const jsonSchema = schemaSpecToJsonSchema(spec)
  539. const prop = jsonSchema.properties['bare'] as Record<string, unknown>
  540. expect(prop).toEqual({ type: 'string' })
  541. expect('description' in prop).toBe(false)
  542. expect('enum' in prop).toBe(false)
  543. expect('default' in prop).toBe(false)
  544. })
  545. it('handles array with no items (items omitted)', () => {
  546. const spec = {
  547. raw: { type: 'array' },
  548. } satisfies SchemaSpec
  549. const jsonSchema = schemaSpecToJsonSchema(spec)
  550. expect(jsonSchema.properties['raw']).toEqual({
  551. type: 'array',
  552. })
  553. })
  554. it('handles nested object with all-optional properties (no required array)', () => {
  555. const spec = {
  556. config: {
  557. type: 'object',
  558. properties: {
  559. host: { type: 'string' },
  560. port: { type: 'number' },
  561. },
  562. },
  563. } satisfies SchemaSpec
  564. const jsonSchema = schemaSpecToJsonSchema(spec)
  565. expect(jsonSchema.properties['config']).toMatchObject({
  566. type: 'object',
  567. properties: {
  568. host: { type: 'string' },
  569. port: { type: 'number' },
  570. },
  571. })
  572. // no 'required' key in the nested object because nothing is required
  573. const config = jsonSchema.properties['config'] as Record<string, unknown>
  574. expect('required' in config).toBe(false)
  575. })
  576. })
  577. describe('schema DSL regressions (Codex review round 2)', () => {
  578. it('InferArgs makes non-required keys genuinely optional (omittable)', () => {
  579. type Args = InferArgs<{
  580. path: { type: 'string'; required: true }
  581. limit: { type: 'number' }
  582. }>
  583. expectTypeOf<Args>().toEqualTypeOf<{ path: string; limit?: number }>()
  584. // omitting the optional key is assignable — the actual regression
  585. const omitted: Args = { path: '/tmp' }
  586. expect(omitted.limit).toBeUndefined()
  587. })
  588. it('InferArgs recurses into array items, including arrays of objects', () => {
  589. type Args = InferArgs<{
  590. names: { type: 'array'; required: true; items: { type: 'string' } }
  591. servers: {
  592. type: 'array'
  593. items: {
  594. type: 'object'
  595. properties: {
  596. host: { type: 'string'; required: true }
  597. port: { type: 'number' }
  598. }
  599. }
  600. }
  601. }>
  602. expectTypeOf<Args>().toEqualTypeOf<{
  603. names: string[]
  604. servers?: { host: string; port?: number }[]
  605. }>()
  606. })
  607. it('runtime JSON Schema matches the array-of-objects inference', () => {
  608. const spec = {
  609. servers: {
  610. type: 'array',
  611. items: {
  612. type: 'object',
  613. properties: {
  614. host: { type: 'string', required: true },
  615. port: { type: 'number' },
  616. },
  617. },
  618. },
  619. } satisfies SchemaSpec
  620. expect(schemaSpecToJsonSchema(spec)).toEqual({
  621. type: 'object',
  622. properties: {
  623. servers: {
  624. type: 'array',
  625. items: {
  626. type: 'object',
  627. properties: {
  628. host: { type: 'string' },
  629. port: { type: 'number' },
  630. },
  631. required: ['host'],
  632. },
  633. },
  634. },
  635. })
  636. })
  637. it('reports messages from non-Error throws (throw { message })', async () => {
  638. const ctx = await setup()
  639. ctx.tools.register({
  640. ...echoTool,
  641. name: 'object-thrower',
  642. async execute() {
  643. // testing non-Error throws on purpose
  644. throw { message: 'denied by object' }
  645. },
  646. })
  647. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'object-thrower', arguments: {} })
  648. expect(result.isError).toBe(true)
  649. expect(result.content[0]).toMatchObject({ text: 'Error: denied by object' })
  650. })
  651. it('reports messages from throws of non-objects (throw "string")', async () => {
  652. const ctx = await setup()
  653. ctx.tools.register({
  654. ...echoTool,
  655. name: 'string-thrower',
  656. async execute() {
  657. // testing primitive throws on purpose
  658. throw 'kaboom'
  659. },
  660. })
  661. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'string-thrower', arguments: {} })
  662. expect(result.isError).toBe(true)
  663. expect(result.content[0]).toMatchObject({ text: 'Error: kaboom' })
  664. })
  665. it('reports messages from throws of objects without message property', async () => {
  666. const ctx = await setup()
  667. ctx.tools.register({
  668. ...echoTool,
  669. name: 'object-no-message',
  670. async execute() {
  671. // testing object throw without .message
  672. throw { code: 500 }
  673. },
  674. })
  675. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'object-no-message', arguments: {} })
  676. expect(result.isError).toBe(true)
  677. const firstContent = result.content[0]!
  678. expect(firstContent.type).toBe('text')
  679. if (firstContent.type === 'text') {
  680. expect(firstContent.text).toBe('Error: [object Object]')
  681. }
  682. })
  683. })
  684. describe('ToolRegistry.get', () => {
  685. it('get() returns the registered tool definition', async () => {
  686. const ctx = await setup()
  687. ctx.tools.register(echoTool)
  688. const tool = ctx.tools.get('echo')
  689. expect(tool).toBeDefined()
  690. expect(tool!.name).toBe('echo')
  691. })
  692. it('get() returns undefined for unknown tool names', async () => {
  693. const ctx = await setup()
  694. expect(ctx.tools.get('nope')).toBeUndefined()
  695. })
  696. })
  697. describe('validateArgs (the runtime-validation RFC, part 1)', () => {
  698. it('returns [] for valid args and is total over malformed input', () => {
  699. const spec = {
  700. path: { type: 'string', required: true },
  701. limit: { type: 'number' },
  702. } satisfies SchemaSpec
  703. expect(validateArgs(spec, { path: '/tmp' })).toEqual([])
  704. expect(validateArgs(spec, { path: '/tmp', limit: 5 })).toEqual([])
  705. // never throws regardless of shape
  706. expect(validateArgs(spec, null)).toHaveLength(1)
  707. expect(validateArgs(spec, 'nope')).toHaveLength(1)
  708. expect(validateArgs(spec, [])).toHaveLength(1)
  709. })
  710. it('flags a missing required key and a required key present as undefined', () => {
  711. const spec = { path: { type: 'string', required: true } } satisfies SchemaSpec
  712. expect(validateArgs(spec, {})).toEqual(['missing required property "path"'])
  713. expect(validateArgs(spec, { path: undefined })).toEqual(['missing required property "path"'])
  714. })
  715. it('allows extra keys (no additionalProperties:false) and omitted optionals', () => {
  716. const spec = { path: { type: 'string', required: true } } satisfies SchemaSpec
  717. expect(validateArgs(spec, { path: '/tmp', extra: 1 })).toEqual([])
  718. })
  719. it('does not apply defaults (validation only)', () => {
  720. const spec = { limit: { type: 'number', default: 25 } } satisfies SchemaSpec
  721. // absent optional is valid, and validation does not synthesize the default
  722. expect(validateArgs(spec, {})).toEqual([])
  723. })
  724. it('type-checks primitives', () => {
  725. const spec = {
  726. s: { type: 'string' },
  727. n: { type: 'number' },
  728. b: { type: 'boolean' },
  729. } satisfies SchemaSpec
  730. expect(validateArgs(spec, { s: 1 })).toEqual(['"s" must be a string'])
  731. expect(validateArgs(spec, { n: 'x' })).toEqual(['"n" must be a number'])
  732. expect(validateArgs(spec, { b: 'x' })).toEqual(['"b" must be a boolean'])
  733. })
  734. it('checks enum membership', () => {
  735. const spec = { color: { type: 'string', enum: ['red', 'green'] } } satisfies SchemaSpec
  736. expect(validateArgs(spec, { color: 'red' })).toEqual([])
  737. expect(validateArgs(spec, { color: 'blue' })).toEqual(['"color" must be one of ["red","green"]'])
  738. })
  739. it('checks enum uniformly with the converter (enum on a non-string prop)', () => {
  740. // The converter emits `enum` regardless of type; the validator must agree.
  741. // `enum` is string[], so a number value can never be a member.
  742. const spec = { n: { type: 'number', enum: ['1', '2'] } } as unknown as SchemaSpec
  743. expect(validateArgs(spec, { n: 1 })).toEqual(['"n" must be one of ["1","2"]'])
  744. })
  745. it('rejects an unknown SchemaType at runtime (assertNever guard)', () => {
  746. const spec = { x: { type: 'weird' } } as unknown as SchemaSpec
  747. expect(() => validateArgs(spec, { x: 1 })).toThrow(/unreachable variant.*validateArgs/)
  748. })
  749. it('recurses into nested objects (and an object without properties only type-checks)', () => {
  750. const spec = {
  751. config: {
  752. type: 'object',
  753. required: true,
  754. properties: { host: { type: 'string', required: true }, port: { type: 'number' } },
  755. },
  756. bag: { type: 'object' },
  757. } satisfies SchemaSpec
  758. expect(validateArgs(spec, { config: { host: 'h' }, bag: { anything: true } })).toEqual([])
  759. expect(validateArgs(spec, { config: { port: 9 }, bag: 5 })).toEqual([
  760. 'missing required property "config.host"',
  761. '"bag" must be an object',
  762. ])
  763. })
  764. it('recurses into array items (and an array without items only type-checks)', () => {
  765. const spec = {
  766. tags: { type: 'array', items: { type: 'string' } },
  767. raw: { type: 'array' },
  768. } satisfies SchemaSpec
  769. expect(validateArgs(spec, { tags: ['a', 'b'], raw: [1, {}, 'x'] })).toEqual([])
  770. expect(validateArgs(spec, { tags: ['a', 2] })).toEqual(['"tags[1]" must be a string'])
  771. // a non-array value for an array-typed prop
  772. expect(validateArgs(spec, { tags: 'nope' })).toEqual(['"tags" must be an array'])
  773. })
  774. it('validates arrays of objects element-wise', () => {
  775. const spec = {
  776. servers: {
  777. type: 'array',
  778. items: { type: 'object', properties: { host: { type: 'string', required: true } } },
  779. },
  780. } satisfies SchemaSpec
  781. expect(validateArgs(spec, { servers: [{ host: 'a' }, {}] })).toEqual([
  782. 'missing required property "servers[1].host"',
  783. ])
  784. })
  785. })
  786. describe('defineTool validation (the runtime-validation RFC, part 1)', () => {
  787. it('returns an isError result with the violations when the model sends bad args', async () => {
  788. const ctx = await setup()
  789. ctx.tools.register(defineTool({
  790. name: 'reader',
  791. description: 'reads a path',
  792. parameters: { path: { type: 'string', required: true } },
  793. async execute(args) {
  794. return [{ type: 'text', text: args.path }]
  795. },
  796. }))
  797. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'reader', arguments: {} })
  798. expect(result.isError).toBe(true)
  799. expect(result.content[0]).toMatchObject({
  800. text: 'Error: invalid arguments: missing required property "path"',
  801. })
  802. })
  803. it('runs execute normally when args are valid', async () => {
  804. const ctx = await setup()
  805. ctx.tools.register(defineTool({
  806. name: 'reader',
  807. description: 'reads a path',
  808. parameters: { path: { type: 'string', required: true } },
  809. async execute(args) {
  810. return [{ type: 'text', text: `read ${args.path}` }]
  811. },
  812. }))
  813. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'reader', arguments: { path: '/x' } })
  814. expect(result).toEqual({ callId: CallId('c1'), content: [{ type: 'text', text: 'read /x' }], isError: false })
  815. })
  816. it('ToolArgsError carries a stable code and the violation list', () => {
  817. const err = new ToolArgsError(['missing required property "a"', '"b" must be a number'])
  818. expect(err).toBeInstanceOf(Error)
  819. expect(err.name).toBe('ToolArgsError')
  820. expect(err.code).toBe('INVALID_ARGS')
  821. expect(err.violations).toEqual(['missing required property "a"', '"b" must be a number'])
  822. expect(err.message).toBe('invalid arguments: missing required property "a"; "b" must be a number')
  823. })
  824. it('a schema-invalid call surfaces the structured error on the result', async () => {
  825. const ctx = await setup()
  826. ctx.tools.register(defineTool({
  827. name: 'reader',
  828. description: 'reads a path',
  829. parameters: { path: { type: 'string', required: true } },
  830. async execute(args) {
  831. return [{ type: 'text', text: args.path }]
  832. },
  833. }))
  834. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'reader', arguments: {} })
  835. expect(result.isError).toBe(true)
  836. expect(result.error).toEqual({ name: 'ToolArgsError', code: 'INVALID_ARGS' })
  837. })
  838. it('a tool throwing a HarnessError surfaces its name and code', async () => {
  839. const { HarnessError } = await import('@deepseek-ai/dsh-llm')
  840. const ctx = await setup()
  841. ctx.tools.register({
  842. ...echoTool,
  843. name: 'coded',
  844. async execute() {
  845. throw new HarnessError('disk full', 'ENOSPC')
  846. },
  847. })
  848. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'coded', arguments: {} })
  849. expect(result.isError).toBe(true)
  850. expect(result.error).toEqual({ name: 'HarnessError', code: 'ENOSPC' })
  851. expect(result.content[0]).toMatchObject({ text: 'Error: disk full' })
  852. })
  853. it('a non-HarnessError throw has no structured error (only the text)', async () => {
  854. const ctx = await setup()
  855. ctx.tools.register({
  856. ...echoTool,
  857. name: 'plain',
  858. async execute() {
  859. throw new Error('just a message')
  860. },
  861. })
  862. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'plain', arguments: {} })
  863. expect(result.isError).toBe(true)
  864. expect(result.error).toBeUndefined()
  865. expect(result.content[0]).toMatchObject({ text: 'Error: just a message' })
  866. })
  867. it('raw-registered tools are NOT validated by defineTool (MCP keeps its own)', async () => {
  868. const ctx = await setup()
  869. // A raw ToolDefinition: no defineTool wrapping, so no validateArgs guard.
  870. ctx.tools.register({
  871. name: 'raw',
  872. description: 'raw tool',
  873. parameters: { type: 'object', properties: { path: { type: 'string' } }, required: ['path'] },
  874. async execute(args: unknown) {
  875. return [{ type: 'text', text: typeof args }]
  876. },
  877. })
  878. // Missing the "required" path — but raw tools validate their own input, so
  879. // this reaches execute rather than being rejected by the harness.
  880. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'raw', arguments: {} })
  881. expect(result.isError).toBe(false)
  882. })
  883. })
  884. describe('defineTool presentation (presentCall / presentResult)', () => {
  885. it('threads presentCall/presentResult onto the ToolDefinition with typed args', () => {
  886. const tool = defineTool({
  887. name: 'demo',
  888. description: 'demo',
  889. parameters: { path: { type: 'string', required: true }, n: { type: 'number' } },
  890. async execute() { return [{ type: 'text', text: 'ok' }] },
  891. presentCall(args) {
  892. // args is typed { path: string; n?: number } — zero casts.
  893. expectTypeOf(args).toEqualTypeOf<{ path: string; n?: number }>()
  894. return { card: 'generic', title: `Open ${args.path}`, kind: 'read', rawInput: args.path }
  895. },
  896. presentResult(args, result) {
  897. return { card: 'generic', title: `Opened ${args.path}`, content: result.content }
  898. },
  899. })
  900. expect(tool.presentCall!({ path: '/a', n: 2 })).toEqual({ card: 'generic', title: 'Open /a', kind: 'read', rawInput: '/a' })
  901. expect(tool.presentResult!({ path: '/a' }, { content: [{ type: 'text', text: 'x' }], isError: false }))
  902. .toEqual({ card: 'generic', title: 'Opened /a', content: [{ type: 'text', text: 'x' }] })
  903. })
  904. it('a tool without presentCall/presentResult leaves them undefined (UI falls back generically)', () => {
  905. const tool = defineTool({
  906. name: 'plain',
  907. description: 'plain',
  908. parameters: { x: { type: 'string', required: true } },
  909. async execute() { return [] },
  910. })
  911. expect(typeof tool.presentCall).toBe('undefined')
  912. expect(typeof tool.presentResult).toBe('undefined')
  913. })
  914. it('presentCall/presentResult validate softly: malformed args return undefined, never throw (display runs on replay)', () => {
  915. const tool = defineTool({
  916. name: 'demo',
  917. description: 'demo',
  918. parameters: { path: { type: 'string', required: true } },
  919. async execute() { return [] },
  920. presentCall: args => ({ card: 'generic', title: args.path }),
  921. presentResult: (args, result) => ({ card: 'generic', title: args.path, content: result.content }),
  922. })
  923. // Unlike execute (which throws ToolArgsError on a mismatch), the display
  924. // methods soft-validate and fall back to undefined so a UI never crashes
  925. // replaying an old/foreign log entry. The ToolDefinition methods take
  926. // `unknown`, so malformed shapes pass without a cast.
  927. expect(tool.presentCall?.({})).toBeUndefined()
  928. expect(tool.presentResult?.({ wrong: 1 }, { content: [], isError: false })).toBeUndefined()
  929. })
  930. })