tools.spec.ts 41 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092
  1. import { describe, expect, expectTypeOf, it } from 'vitest'
  2. import { Context } from 'cordis'
  3. import { CallId, HarnessError } from '@deepseek-ai/dsh-llm'
  4. import SystemPrompt from '@deepseek-ai/dsh-system-prompt'
  5. import ToolRegistry, {
  6. defineTool, schemaSpecToJsonSchema, validateArgs, ToolArgsError, ToolNotFoundError,
  7. type InferArgs, type SchemaSpec, type PreToolDecision, type PostToolDecision,
  8. } from '@deepseek-ai/dsh-tools'
  9. async function setup() {
  10. const ctx = new Context()
  11. await ctx.plugin(SystemPrompt)
  12. await ctx.plugin(ToolRegistry)
  13. return ctx
  14. }
  15. const echoTool = defineTool({
  16. name: 'echo',
  17. description: 'echo arguments back',
  18. parameters: { text: { type: 'string' } },
  19. async execute(args) {
  20. return [{ type: 'text' as const, text: args.text ?? '' }]
  21. },
  22. })
  23. describe('ToolRegistry', () => {
  24. it('registers tools, exposes schemas, and feeds the system-prompt assembly', async () => {
  25. const ctx = await setup()
  26. ctx.tools.register(echoTool)
  27. expect(ctx.tools.schemas()).toEqual([{
  28. name: 'echo',
  29. description: 'echo arguments back',
  30. parameters: { type: 'object', properties: { text: { type: 'string' } } },
  31. }])
  32. // schemas() result must not leak execute — ToolSchema deliberately has no
  33. // 'execute' key, so widen through unknown to probe for the absent property
  34. expect((ctx.tools.schemas()[0] as unknown as Record<string, unknown>).execute).toBeUndefined()
  35. const assembly = await ctx.systemPrompt.assemble()
  36. expect(assembly.tools.map(t => t.name)).toEqual(['echo'])
  37. })
  38. it('schemas() drops the UI presentation callbacks — they must never reach the model', async () => {
  39. const ctx = await setup()
  40. // A tool that declares presentCall/presentResult (functions). schemas() feeds
  41. // the system-prompt assembly → the model request, so those callbacks (and
  42. // `execute`) must be stripped: a function in the JSON tool schema would
  43. // corrupt the request. schemas() is an explicit allowlist, so it can't leak.
  44. ctx.tools.register(defineTool({
  45. name: 'present',
  46. description: 'has presenters',
  47. parameters: { x: { type: 'string', required: true } },
  48. async execute() { return [] },
  49. presentCall: args => ({ card: 'generic', title: args.x }),
  50. presentResult: (args, result) => ({ card: 'generic', title: args.x, content: result.content }),
  51. }))
  52. const schema = ctx.tools.schemas()[0] as unknown as Record<string, unknown>
  53. expect(Object.keys(schema).sort()).toEqual(['description', 'name', 'parameters'])
  54. expect(schema.presentCall).toBeUndefined()
  55. expect(schema.presentResult).toBeUndefined()
  56. expect(schema.execute).toBeUndefined()
  57. })
  58. it('schemas() preserves `strict` when set (allowlist keeps the model-facing fields)', async () => {
  59. const ctx = await setup()
  60. ctx.tools.register(defineTool({
  61. name: 'strict-tool',
  62. description: 'd',
  63. parameters: { x: { type: 'string', required: true } },
  64. strict: true,
  65. async execute() { return [] },
  66. }))
  67. expect(ctx.tools.schemas()[0]).toMatchObject({ name: 'strict-tool', strict: true })
  68. })
  69. it('executes a tool and returns its content', async () => {
  70. const ctx = await setup()
  71. ctx.tools.register(echoTool)
  72. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'echo', arguments: { text: 'hi' } })
  73. expect(result).toEqual({ callId: CallId('c1'), content: [{ type: 'text', text: 'hi' }], isError: false })
  74. })
  75. it('threads a tool-attached meta (object return form) onto the result', async () => {
  76. const ctx = await setup()
  77. ctx.tools.register({
  78. ...echoTool,
  79. name: 'meta-tool',
  80. async execute() {
  81. return { content: [{ type: 'text', text: 'ok' }], meta: { diffs: [{ path: 'a', oldText: null, newText: 'x' }] } }
  82. },
  83. })
  84. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'meta-tool', arguments: {} })
  85. expect(result).toEqual({
  86. callId: CallId('c1'),
  87. content: [{ type: 'text', text: 'ok' }],
  88. isError: false,
  89. meta: { diffs: [{ path: 'a', oldText: null, newText: 'x' }] },
  90. })
  91. })
  92. it('omits meta when the object return form supplies none', async () => {
  93. const ctx = await setup()
  94. ctx.tools.register({
  95. ...echoTool,
  96. name: 'no-meta-tool',
  97. async execute() {
  98. return { content: [{ type: 'text', text: 'ok' }] }
  99. },
  100. })
  101. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'no-meta-tool', arguments: {} })
  102. expect(result).toEqual({ callId: CallId('c1'), content: [{ type: 'text', text: 'ok' }], isError: false })
  103. expect('meta' in result).toBe(false)
  104. })
  105. it('returns isError results for unknown tools and throwing tools', async () => {
  106. const ctx = await setup()
  107. ctx.tools.register({
  108. ...echoTool,
  109. name: 'boom',
  110. async execute() {
  111. throw new Error('exploded')
  112. },
  113. })
  114. const unknown = await ctx.tools.execute({ callId: CallId('c1'), name: 'nope', arguments: {} })
  115. expect(unknown.isError).toBe(true)
  116. expect(unknown.content[0]).toMatchObject({ text: 'Error: unknown tool "nope"' })
  117. // An unknown tool is a routable failure class, same as a tool-thrown one.
  118. expect(unknown.error).toEqual({ name: 'ToolNotFoundError', code: 'UNKNOWN_TOOL' })
  119. const thrown = await ctx.tools.execute({ callId: CallId('c2'), name: 'boom', arguments: {} })
  120. expect(thrown.isError).toBe(true)
  121. expect(thrown.content[0]).toMatchObject({ text: 'Error: exploded' })
  122. })
  123. it('ToolNotFoundError carries the tool name and a stable code', async () => {
  124. const { HarnessError } = await import('@deepseek-ai/dsh-llm')
  125. const err = new ToolNotFoundError('ghost')
  126. expect(err).toBeInstanceOf(HarnessError)
  127. expect(err.name).toBe('ToolNotFoundError')
  128. expect(err.code).toBe('UNKNOWN_TOOL')
  129. expect(err.toolName).toBe('ghost')
  130. expect(err.message).toBe('unknown tool "ghost"')
  131. })
  132. it('lets a tools/pre-execute listener deny a call (permission pattern)', async () => {
  133. const ctx = await setup()
  134. ctx.tools.register(echoTool)
  135. ctx.on('tools/pre-execute', async (exec, next): Promise<PreToolDecision> => {
  136. if (exec.name === 'echo') return { kind: 'deny', reason: 'denied by policy' }
  137. return next()
  138. })
  139. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'echo', arguments: { text: 'hi' } })
  140. expect(result.isError).toBe(true)
  141. expect(result.content[0]).toMatchObject({ text: 'Error: denied by policy' })
  142. })
  143. it('an ask decision degrades to deny until the permission system lands', async () => {
  144. const ctx = await setup()
  145. ctx.tools.register(echoTool)
  146. ctx.on('tools/pre-execute', async (_exec, _next): Promise<PreToolDecision> =>
  147. ({ kind: 'ask', reason: 'needs approval' }))
  148. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'echo', arguments: { text: 'hi' } })
  149. expect(result.isError).toBe(true)
  150. expect(result.content[0]).toMatchObject({ text: 'Error: needs approval' })
  151. })
  152. it('an ask decision with no reason degrades to deny with a default message', async () => {
  153. const ctx = await setup()
  154. ctx.tools.register(echoTool)
  155. ctx.on('tools/pre-execute', async (_exec, _next): Promise<PreToolDecision> => ({ kind: 'ask' }))
  156. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'echo', arguments: { text: 'hi' } })
  157. expect(result.isError).toBe(true)
  158. expect(result.content[0]).toMatchObject({ text: 'Error: tool "echo" requires approval (not yet supported)' })
  159. })
  160. it('a tools/post-execute listener can replace the result content (accept) ', async () => {
  161. const ctx = await setup()
  162. ctx.tools.register(echoTool)
  163. ctx.on('tools/post-execute', async (_exec, _result, _next): Promise<PostToolDecision> =>
  164. ({ kind: 'accept', content: [{ type: 'text', text: 'rewritten' }] }))
  165. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'echo', arguments: { text: 'hi' } })
  166. expect(result.isError).toBe(false)
  167. expect(result.content[0]).toMatchObject({ text: 'rewritten' })
  168. })
  169. it('a tools/post-execute block turns the call into an isError with corrective feedback', async () => {
  170. const ctx = await setup()
  171. ctx.tools.register(echoTool)
  172. ctx.on('tools/post-execute', async (_exec, _result, _next): Promise<PostToolDecision> =>
  173. ({ kind: 'block', feedback: [{ type: 'text', text: 'output rejected: try again' }] }))
  174. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'echo', arguments: { text: 'hi' } })
  175. expect(result.isError).toBe(true)
  176. expect(result.content[0]).toMatchObject({ text: 'output rejected: try again' })
  177. })
  178. it('a block decision can ALSO attach additionalContext', async () => {
  179. const ctx = await setup()
  180. ctx.tools.register(echoTool)
  181. ctx.on('tools/post-execute', async (_exec, _result, _next): Promise<PostToolDecision> =>
  182. ({
  183. kind: 'block',
  184. feedback: [{ type: 'text', text: 'rejected' }],
  185. additionalContext: { content: [{ type: 'text', text: 'why it was rejected' }], source: { kind: 'plugin', plugin: 'test' } },
  186. }))
  187. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'echo', arguments: { text: 'hi' } })
  188. expect(result.isError).toBe(true)
  189. expect(result.content[0]).toMatchObject({ text: 'rejected' })
  190. expect(result.additionalContext).toMatchObject({ content: [{ text: 'why it was rejected' }], source: { kind: 'plugin', plugin: 'test' } })
  191. })
  192. it('a post-execute additionalContext rides on the result for the loop to buffer', async () => {
  193. const ctx = await setup()
  194. ctx.tools.register(echoTool)
  195. ctx.on('tools/post-execute', async (_exec, _result, _next): Promise<PostToolDecision> =>
  196. ({ kind: 'accept', additionalContext: { content: [{ type: 'text', text: 'fyi' }], source: { kind: 'plugin', plugin: 'test' } } }))
  197. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'echo', arguments: { text: 'hi' } })
  198. expect(result.additionalContext).toMatchObject({ content: [{ text: 'fyi' }], source: { kind: 'plugin', plugin: 'test' } })
  199. })
  200. it('a post-execute listener mutating the result object cannot corrupt callId/isError/error', async () => {
  201. // The decision is the ONLY sanctioned channel to change the outcome. A
  202. // listener that reaches in and mutates the passed result reference (flipping
  203. // isError, rewriting callId, attaching a bogus error) must NOT affect what
  204. // execute() returns — the registry snapshots the authoritative fields before
  205. // the waterfall and rebuilds from the snapshot + decision.
  206. const ctx = await setup()
  207. ctx.tools.register(echoTool)
  208. ctx.on('tools/post-execute', async (_exec, result, next) => {
  209. const mutable = result as { callId: string; isError: boolean; error?: unknown; content: unknown[] }
  210. mutable.callId = 'hijacked'
  211. mutable.isError = true
  212. mutable.error = { name: 'Evil', code: 'EVIL' }
  213. mutable.content.push({ type: 'text', text: 'INJECTED' }) // in-place array mutation
  214. return next() // delegate to the default accept — no decision-level override
  215. })
  216. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'echo', arguments: { text: 'hi' } })
  217. expect(result.callId).toBe(CallId('c1')) // authoritative exec.callId, not 'hijacked'
  218. expect(result.isError).toBe(false) // the real (successful) dispatch outcome
  219. expect(result.error).toBeUndefined() // no listener-injected error
  220. expect(result.content).toHaveLength(1) // the in-place push did not leak in
  221. expect(result.content[0]).toMatchObject({ text: 'hi' })
  222. expect(result.content.some(b => (b as { text?: string }).text === 'INJECTED')).toBe(false)
  223. })
  224. it('composes pre + post waterfalls around dispatch (sandbox-wrap pattern)', async () => {
  225. const ctx = await setup()
  226. ctx.tools.register(echoTool)
  227. const order: string[] = []
  228. ctx.on('tools/pre-execute', async (_exec, next) => {
  229. order.push('pre:before')
  230. const decision = await next()
  231. order.push('pre:after')
  232. return decision
  233. })
  234. ctx.on('tools/post-execute', async (_exec, _result, next) => {
  235. order.push('post:before')
  236. const decision = await next()
  237. order.push('post:after')
  238. return decision
  239. })
  240. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'echo', arguments: { text: 'x' } })
  241. expect(result.isError).toBe(false)
  242. // pre runs fully (gate) before dispatch, then post runs over the result.
  243. expect(order).toEqual(['pre:before', 'pre:after', 'post:before', 'post:after'])
  244. })
  245. it('returns an isError result when a tools/pre-execute listener throws', async () => {
  246. const ctx = await setup()
  247. ctx.tools.register(echoTool)
  248. ctx.on('tools/pre-execute', async () => {
  249. throw new Error('permission hook broke')
  250. })
  251. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'echo', arguments: { text: 'hi' } })
  252. expect(result).toEqual({
  253. callId: CallId('c1'),
  254. content: [{ type: 'text', text: 'Error: permission hook broke' }],
  255. isError: true,
  256. })
  257. })
  258. it('returns an isError result when a tools/post-execute listener throws', async () => {
  259. const ctx = await setup()
  260. ctx.tools.register(echoTool)
  261. ctx.on('tools/post-execute', async () => {
  262. throw new Error('post hook broke')
  263. })
  264. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'echo', arguments: { text: 'hi' } })
  265. expect(result).toEqual({
  266. callId: CallId('c1'),
  267. content: [{ type: 'text', text: 'Error: post hook broke' }],
  268. isError: true,
  269. })
  270. })
  271. it('preserves structured error info when a tools/pre-execute listener throws HarnessError', async () => {
  272. const ctx = await setup()
  273. ctx.tools.register(echoTool)
  274. ctx.on('tools/pre-execute', async () => {
  275. throw new HarnessError('denied', 'DENIED')
  276. })
  277. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'echo', arguments: { text: 'hi' } })
  278. expect(result).toMatchObject({
  279. callId: CallId('c1'),
  280. isError: true,
  281. error: { name: 'HarnessError', code: 'DENIED' },
  282. })
  283. })
  284. it('schemas() snapshots tool schemas instead of exposing registry objects', async () => {
  285. const ctx = await setup()
  286. ctx.tools.register(echoTool)
  287. const first = ctx.tools.schemas()
  288. const firstParameters = first[0]!.parameters as { properties: Record<string, unknown> }
  289. firstParameters.properties['mutated'] = { type: 'string' }
  290. first[0]!.description = 'mutated'
  291. expect(ctx.tools.schemas()).toEqual([{
  292. name: 'echo',
  293. description: 'echo arguments back',
  294. parameters: { type: 'object', properties: { text: { type: 'string' } } },
  295. }])
  296. })
  297. it('rejects duplicate names and unregisters on fiber dispose (HMR safety)', async () => {
  298. const ctx = await setup()
  299. ctx.tools.register(echoTool)
  300. expect(() => ctx.tools.register(echoTool)).toThrow('already registered')
  301. const fiber = await ctx.plugin(Object.assign((inner: Context) => {
  302. inner.tools.register({ ...echoTool, name: 'scoped' })
  303. }, { inject: ['tools'] }))
  304. expect(ctx.tools.schemas().map(t => t.name)).toEqual(['echo', 'scoped'])
  305. await fiber.dispose()
  306. expect(ctx.tools.schemas().map(t => t.name)).toEqual(['echo'])
  307. })
  308. it('returns a callable disposer from register() that unregisters the tool', async () => {
  309. const ctx = await setup()
  310. ctx.tools.register(echoTool)
  311. // Register a second tool and call its returned disposer directly
  312. const dispose = ctx.tools.register({ ...echoTool, name: 'disposable' })
  313. expect(ctx.tools.schemas().map(t => t.name)).toEqual(['echo', 'disposable'])
  314. dispose()
  315. expect(ctx.tools.schemas().map(t => t.name)).toEqual(['echo'])
  316. })
  317. it('rolls back the tool entry when a tools/change listener throws (P1-1)', async () => {
  318. const ctx = await setup()
  319. let threw = false
  320. ctx.on('tools/change', () => {
  321. if (!threw) { threw = true; throw new Error('boom change listener') }
  322. })
  323. // The throwing emit must roll the entry back, not leak it.
  324. expect(() => ctx.tools.register(echoTool)).toThrow('boom change listener')
  325. expect(ctx.tools.get('echo')).toBeUndefined() // rolled back, not leaked
  326. expect(ctx.tools.schemas()).toHaveLength(0)
  327. // A subsequent listener-free register of the SAME name succeeds and is
  328. // exposed exactly once (the duplicate-name check is not wedged).
  329. const dispose = ctx.tools.register(echoTool)
  330. expect(ctx.tools.schemas().map(t => t.name)).toEqual(['echo'])
  331. dispose()
  332. expect(ctx.tools.get('echo')).toBeUndefined()
  333. })
  334. })
  335. describe('defineTool / schema DSL', () => {
  336. it('converts SchemaSpec to standard JSON Schema with required array', () => {
  337. const spec = {
  338. path: { type: 'string', required: true, description: 'Absolute path' },
  339. offset: { type: 'number' },
  340. limit: { type: 'number', description: 'Max lines' },
  341. } satisfies SchemaSpec
  342. const jsonSchema = schemaSpecToJsonSchema(spec)
  343. expect(jsonSchema).toEqual({
  344. type: 'object',
  345. properties: {
  346. path: { type: 'string', description: 'Absolute path' },
  347. offset: { type: 'number' },
  348. limit: { type: 'number', description: 'Max lines' },
  349. },
  350. required: ['path'],
  351. })
  352. })
  353. it('handles empty spec (no properties, no required)', () => {
  354. expect(schemaSpecToJsonSchema({})).toEqual({
  355. type: 'object',
  356. properties: {},
  357. })
  358. })
  359. it('handles nested object spec', () => {
  360. const spec = {
  361. config: {
  362. type: 'object',
  363. required: true,
  364. properties: {
  365. host: { type: 'string', required: true },
  366. port: { type: 'number' },
  367. },
  368. },
  369. } satisfies SchemaSpec
  370. const jsonSchema = schemaSpecToJsonSchema(spec)
  371. expect(jsonSchema).toEqual({
  372. type: 'object',
  373. properties: {
  374. config: {
  375. type: 'object',
  376. properties: {
  377. host: { type: 'string' },
  378. port: { type: 'number' },
  379. },
  380. required: ['host'],
  381. },
  382. },
  383. required: ['config'],
  384. })
  385. })
  386. it('defineTool returns a valid ToolDefinition with typed execute', async () => {
  387. const ctx = await setup()
  388. const tool = defineTool({
  389. name: 'typed-echo',
  390. description: 'A typed echo tool',
  391. parameters: {
  392. text: { type: 'string', required: true },
  393. uppercase: { type: 'boolean' },
  394. },
  395. async execute(args) {
  396. // args is typed: { text: string; uppercase?: boolean }
  397. const result = args.uppercase ? args.text.toUpperCase() : args.text
  398. return [{ type: 'text', text: result }]
  399. },
  400. })
  401. ctx.tools.register(tool)
  402. expect(ctx.tools.schemas()).toEqual([{
  403. name: 'typed-echo',
  404. description: 'A typed echo tool',
  405. parameters: {
  406. type: 'object',
  407. properties: {
  408. text: { type: 'string' },
  409. uppercase: { type: 'boolean' },
  410. },
  411. required: ['text'],
  412. },
  413. }])
  414. const result = await ctx.tools.execute({
  415. callId: CallId('c1'),
  416. name: 'typed-echo',
  417. arguments: { text: 'hello', uppercase: true },
  418. })
  419. expect(result.isError).toBe(false)
  420. expect(result.content).toEqual([{ type: 'text', text: 'HELLO' }])
  421. })
  422. it('type-level: InferArgs maps required properties to non-optional', () => {
  423. // Compile-time check: if this compiles, InferArgs is correct.
  424. // args.a is string (required), args.b is number|undefined (optional).
  425. const tool = defineTool({
  426. name: 'type-check',
  427. description: '',
  428. parameters: { a: { type: 'string' as const, required: true as const }, b: { type: 'number' as const } },
  429. async execute(args) {
  430. // Verify types at runtime via typeof
  431. expect(typeof args.a).toBe('string')
  432. // args.b should be undefined when not provided
  433. void args
  434. return [{ type: 'text', text: args.a }]
  435. },
  436. })
  437. void tool
  438. })
  439. it('registry round-trips a defineTool definition (register→schemas→execute)', async () => {
  440. const ctx = await setup()
  441. ctx.tools.register(defineTool({
  442. name: 'roundtrip',
  443. description: 'Round-trip test',
  444. parameters: {
  445. req: { type: 'string', required: true },
  446. opt: { type: 'number', description: 'Optional number' },
  447. },
  448. async execute(args) {
  449. return [{ type: 'text', text: `${args.req}:${args.opt ?? 'none'}` }]
  450. },
  451. }))
  452. // Schema round-trip: schemas() returns standard JSON Schema
  453. const schemas = ctx.tools.schemas()
  454. expect(schemas).toHaveLength(1)
  455. expect(schemas[0]!.parameters).toEqual({
  456. type: 'object',
  457. properties: {
  458. req: { type: 'string' },
  459. opt: { type: 'number', description: 'Optional number' },
  460. },
  461. required: ['req'],
  462. })
  463. // Execution round-trip
  464. const result = await ctx.tools.execute({
  465. callId: CallId('c1'),
  466. name: 'roundtrip',
  467. arguments: { req: 'hello' },
  468. })
  469. expect(result.isError).toBe(false)
  470. expect(result.content).toEqual([{ type: 'text', text: 'hello:none' }])
  471. })
  472. it('still accepts raw JSON-Schema ToolDefinition directly (MCP interop)', async () => {
  473. const ctx = await setup()
  474. ctx.tools.register({
  475. name: 'raw-tool',
  476. description: 'Raw JSON Schema tool (like an MCP adapter would register)',
  477. parameters: {
  478. type: 'object',
  479. properties: { path: { type: 'string' } },
  480. required: ['path'],
  481. },
  482. async execute(args: unknown) {
  483. const p = args as { path: string }
  484. return [{ type: 'text', text: p.path }]
  485. },
  486. })
  487. const schemas = ctx.tools.schemas()
  488. expect(schemas[0]!.parameters).toEqual({
  489. type: 'object',
  490. properties: { path: { type: 'string' } },
  491. required: ['path'],
  492. })
  493. const result = await ctx.tools.execute({
  494. callId: CallId('c1'),
  495. name: 'raw-tool',
  496. arguments: { path: '/tmp' },
  497. })
  498. expect(result.isError).toBe(false)
  499. expect(result.content).toEqual([{ type: 'text', text: '/tmp' }])
  500. })
  501. })
  502. describe('schema DSL edge cases', () => {
  503. it('emits enum values in JSON Schema property', () => {
  504. const spec = {
  505. color: { type: 'string', enum: ['red', 'green', 'blue'], description: 'Color choice' },
  506. } satisfies SchemaSpec
  507. const jsonSchema = schemaSpecToJsonSchema(spec)
  508. expect(jsonSchema.properties['color']).toMatchObject({
  509. type: 'string',
  510. enum: ['red', 'green', 'blue'],
  511. description: 'Color choice',
  512. })
  513. })
  514. it('emits default value in JSON Schema property', () => {
  515. const spec = {
  516. limit: { type: 'number', default: 25 },
  517. } satisfies SchemaSpec
  518. const jsonSchema = schemaSpecToJsonSchema(spec)
  519. expect(jsonSchema.properties['limit']).toMatchObject({
  520. type: 'number',
  521. default: 25,
  522. })
  523. })
  524. it('handles array items without nested properties (plain type array)', () => {
  525. const spec = {
  526. tags: { type: 'array', items: { type: 'string' } },
  527. } satisfies SchemaSpec
  528. const jsonSchema = schemaSpecToJsonSchema(spec)
  529. expect(jsonSchema.properties['tags']).toEqual({
  530. type: 'array',
  531. items: { type: 'string' },
  532. })
  533. })
  534. it('defineTool passes through strict flag when set to true', () => {
  535. const tool = defineTool({
  536. name: 'strict-tool',
  537. description: 'A strict tool',
  538. parameters: { input: { type: 'string' } },
  539. strict: true,
  540. async execute(args) {
  541. return [{ type: 'text' as const, text: args.input ?? '' }]
  542. },
  543. })
  544. expect(tool.strict).toBe(true)
  545. })
  546. it('defineTool omits strict when not provided', () => {
  547. const tool = defineTool({
  548. name: 'non-strict-tool',
  549. description: 'A non-strict tool',
  550. parameters: { input: { type: 'string' } },
  551. async execute(args) {
  552. return [{ type: 'text' as const, text: args.input ?? '' }]
  553. },
  554. })
  555. expect('strict' in tool).toBe(false)
  556. })
  557. it('defineTool strict=false is included', () => {
  558. const tool = defineTool({
  559. name: 'explicitly-non-strict',
  560. description: 'Explicitly non-strict',
  561. parameters: { input: { type: 'string' } },
  562. strict: false,
  563. async execute(args) {
  564. return [{ type: 'text' as const, text: args.input ?? '' }]
  565. },
  566. })
  567. expect(tool.strict).toBe(false)
  568. })
  569. it('handles enum and default together in one property', () => {
  570. const spec = {
  571. level: { type: 'string', enum: ['low', 'high'], default: 'low' },
  572. } satisfies SchemaSpec
  573. const jsonSchema = schemaSpecToJsonSchema(spec)
  574. expect(jsonSchema.properties['level']).toMatchObject({
  575. type: 'string',
  576. enum: ['low', 'high'],
  577. default: 'low',
  578. })
  579. })
  580. it('omits description, enum, default keys when not specified', () => {
  581. const spec = {
  582. bare: { type: 'string' },
  583. } satisfies SchemaSpec
  584. const jsonSchema = schemaSpecToJsonSchema(spec)
  585. const prop = jsonSchema.properties['bare'] as Record<string, unknown>
  586. expect(prop).toEqual({ type: 'string' })
  587. expect('description' in prop).toBe(false)
  588. expect('enum' in prop).toBe(false)
  589. expect('default' in prop).toBe(false)
  590. })
  591. it('handles array with no items (items omitted)', () => {
  592. const spec = {
  593. raw: { type: 'array' },
  594. } satisfies SchemaSpec
  595. const jsonSchema = schemaSpecToJsonSchema(spec)
  596. expect(jsonSchema.properties['raw']).toEqual({
  597. type: 'array',
  598. })
  599. })
  600. it('handles nested object with all-optional properties (no required array)', () => {
  601. const spec = {
  602. config: {
  603. type: 'object',
  604. properties: {
  605. host: { type: 'string' },
  606. port: { type: 'number' },
  607. },
  608. },
  609. } satisfies SchemaSpec
  610. const jsonSchema = schemaSpecToJsonSchema(spec)
  611. expect(jsonSchema.properties['config']).toMatchObject({
  612. type: 'object',
  613. properties: {
  614. host: { type: 'string' },
  615. port: { type: 'number' },
  616. },
  617. })
  618. // no 'required' key in the nested object because nothing is required
  619. const config = jsonSchema.properties['config'] as Record<string, unknown>
  620. expect('required' in config).toBe(false)
  621. })
  622. })
  623. describe('schema DSL regressions (Codex review round 2)', () => {
  624. it('InferArgs makes non-required keys genuinely optional (omittable)', () => {
  625. type Args = InferArgs<{
  626. path: { type: 'string'; required: true }
  627. limit: { type: 'number' }
  628. }>
  629. expectTypeOf<Args>().toEqualTypeOf<{ path: string; limit?: number }>()
  630. // omitting the optional key is assignable — the actual regression
  631. const omitted: Args = { path: '/tmp' }
  632. expect(omitted.limit).toBeUndefined()
  633. })
  634. it('InferArgs recurses into array items, including arrays of objects', () => {
  635. type Args = InferArgs<{
  636. names: { type: 'array'; required: true; items: { type: 'string' } }
  637. servers: {
  638. type: 'array'
  639. items: {
  640. type: 'object'
  641. properties: {
  642. host: { type: 'string'; required: true }
  643. port: { type: 'number' }
  644. }
  645. }
  646. }
  647. }>
  648. expectTypeOf<Args>().toEqualTypeOf<{
  649. names: string[]
  650. servers?: { host: string; port?: number }[]
  651. }>()
  652. })
  653. it('runtime JSON Schema matches the array-of-objects inference', () => {
  654. const spec = {
  655. servers: {
  656. type: 'array',
  657. items: {
  658. type: 'object',
  659. properties: {
  660. host: { type: 'string', required: true },
  661. port: { type: 'number' },
  662. },
  663. },
  664. },
  665. } satisfies SchemaSpec
  666. expect(schemaSpecToJsonSchema(spec)).toEqual({
  667. type: 'object',
  668. properties: {
  669. servers: {
  670. type: 'array',
  671. items: {
  672. type: 'object',
  673. properties: {
  674. host: { type: 'string' },
  675. port: { type: 'number' },
  676. },
  677. required: ['host'],
  678. },
  679. },
  680. },
  681. })
  682. })
  683. it('reports messages from non-Error throws (throw { message })', async () => {
  684. const ctx = await setup()
  685. ctx.tools.register({
  686. ...echoTool,
  687. name: 'object-thrower',
  688. async execute() {
  689. // testing non-Error throws on purpose
  690. throw { message: 'denied by object' }
  691. },
  692. })
  693. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'object-thrower', arguments: {} })
  694. expect(result.isError).toBe(true)
  695. expect(result.content[0]).toMatchObject({ text: 'Error: denied by object' })
  696. })
  697. it('reports messages from throws of non-objects (throw "string")', async () => {
  698. const ctx = await setup()
  699. ctx.tools.register({
  700. ...echoTool,
  701. name: 'string-thrower',
  702. async execute() {
  703. // testing primitive throws on purpose
  704. throw 'kaboom'
  705. },
  706. })
  707. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'string-thrower', arguments: {} })
  708. expect(result.isError).toBe(true)
  709. expect(result.content[0]).toMatchObject({ text: 'Error: kaboom' })
  710. })
  711. it('reports messages from throws of objects without message property', async () => {
  712. const ctx = await setup()
  713. ctx.tools.register({
  714. ...echoTool,
  715. name: 'object-no-message',
  716. async execute() {
  717. // testing object throw without .message
  718. throw { code: 500 }
  719. },
  720. })
  721. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'object-no-message', arguments: {} })
  722. expect(result.isError).toBe(true)
  723. const firstContent = result.content[0]!
  724. expect(firstContent.type).toBe('text')
  725. if (firstContent.type === 'text') {
  726. expect(firstContent.text).toBe('Error: [object Object]')
  727. }
  728. })
  729. })
  730. describe('ToolRegistry.get', () => {
  731. it('get() returns the registered tool definition', async () => {
  732. const ctx = await setup()
  733. ctx.tools.register(echoTool)
  734. const tool = ctx.tools.get('echo')
  735. expect(tool).toBeDefined()
  736. expect(tool!.name).toBe('echo')
  737. })
  738. it('get() returns undefined for unknown tool names', async () => {
  739. const ctx = await setup()
  740. expect(ctx.tools.get('nope')).toBeUndefined()
  741. })
  742. })
  743. describe('validateArgs (the runtime-validation RFC, part 1)', () => {
  744. it('returns [] for valid args and is total over malformed input', () => {
  745. const spec = {
  746. path: { type: 'string', required: true },
  747. limit: { type: 'number' },
  748. } satisfies SchemaSpec
  749. expect(validateArgs(spec, { path: '/tmp' })).toEqual([])
  750. expect(validateArgs(spec, { path: '/tmp', limit: 5 })).toEqual([])
  751. // never throws regardless of shape
  752. expect(validateArgs(spec, null)).toHaveLength(1)
  753. expect(validateArgs(spec, 'nope')).toHaveLength(1)
  754. expect(validateArgs(spec, [])).toHaveLength(1)
  755. })
  756. it('flags a missing required key and a required key present as undefined', () => {
  757. const spec = { path: { type: 'string', required: true } } satisfies SchemaSpec
  758. expect(validateArgs(spec, {})).toEqual(['missing required property "path"'])
  759. expect(validateArgs(spec, { path: undefined })).toEqual(['missing required property "path"'])
  760. })
  761. it('allows extra keys (no additionalProperties:false) and omitted optionals', () => {
  762. const spec = { path: { type: 'string', required: true } } satisfies SchemaSpec
  763. expect(validateArgs(spec, { path: '/tmp', extra: 1 })).toEqual([])
  764. })
  765. it('does not apply defaults (validation only)', () => {
  766. const spec = { limit: { type: 'number', default: 25 } } satisfies SchemaSpec
  767. // absent optional is valid, and validation does not synthesize the default
  768. expect(validateArgs(spec, {})).toEqual([])
  769. })
  770. it('type-checks primitives', () => {
  771. const spec = {
  772. s: { type: 'string' },
  773. n: { type: 'number' },
  774. b: { type: 'boolean' },
  775. } satisfies SchemaSpec
  776. expect(validateArgs(spec, { s: 1 })).toEqual(['"s" must be a string'])
  777. expect(validateArgs(spec, { n: 'x' })).toEqual(['"n" must be a number'])
  778. expect(validateArgs(spec, { b: 'x' })).toEqual(['"b" must be a boolean'])
  779. })
  780. it('checks enum membership', () => {
  781. const spec = { color: { type: 'string', enum: ['red', 'green'] } } satisfies SchemaSpec
  782. expect(validateArgs(spec, { color: 'red' })).toEqual([])
  783. expect(validateArgs(spec, { color: 'blue' })).toEqual(['"color" must be one of ["red","green"]'])
  784. })
  785. it('checks enum uniformly with the converter (enum on a non-string prop)', () => {
  786. // The converter emits `enum` regardless of type; the validator must agree.
  787. // `enum` is string[], so a number value can never be a member.
  788. const spec = { n: { type: 'number', enum: ['1', '2'] } } as unknown as SchemaSpec
  789. expect(validateArgs(spec, { n: 1 })).toEqual(['"n" must be one of ["1","2"]'])
  790. })
  791. it('rejects an unknown SchemaType at runtime (assertNever guard)', () => {
  792. const spec = { x: { type: 'weird' } } as unknown as SchemaSpec
  793. expect(() => validateArgs(spec, { x: 1 })).toThrow(/unreachable variant.*validateArgs/)
  794. })
  795. it('recurses into nested objects (and an object without properties only type-checks)', () => {
  796. const spec = {
  797. config: {
  798. type: 'object',
  799. required: true,
  800. properties: { host: { type: 'string', required: true }, port: { type: 'number' } },
  801. },
  802. bag: { type: 'object' },
  803. } satisfies SchemaSpec
  804. expect(validateArgs(spec, { config: { host: 'h' }, bag: { anything: true } })).toEqual([])
  805. expect(validateArgs(spec, { config: { port: 9 }, bag: 5 })).toEqual([
  806. 'missing required property "config.host"',
  807. '"bag" must be an object',
  808. ])
  809. })
  810. it('recurses into array items (and an array without items only type-checks)', () => {
  811. const spec = {
  812. tags: { type: 'array', items: { type: 'string' } },
  813. raw: { type: 'array' },
  814. } satisfies SchemaSpec
  815. expect(validateArgs(spec, { tags: ['a', 'b'], raw: [1, {}, 'x'] })).toEqual([])
  816. expect(validateArgs(spec, { tags: ['a', 2] })).toEqual(['"tags[1]" must be a string'])
  817. // a non-array value for an array-typed prop
  818. expect(validateArgs(spec, { tags: 'nope' })).toEqual(['"tags" must be an array'])
  819. })
  820. it('validates arrays of objects element-wise', () => {
  821. const spec = {
  822. servers: {
  823. type: 'array',
  824. items: { type: 'object', properties: { host: { type: 'string', required: true } } },
  825. },
  826. } satisfies SchemaSpec
  827. expect(validateArgs(spec, { servers: [{ host: 'a' }, {}] })).toEqual([
  828. 'missing required property "servers[1].host"',
  829. ])
  830. })
  831. })
  832. describe('defineTool validation (the runtime-validation RFC, part 1)', () => {
  833. it('returns an isError result with the violations when the model sends bad args', async () => {
  834. const ctx = await setup()
  835. ctx.tools.register(defineTool({
  836. name: 'reader',
  837. description: 'reads a path',
  838. parameters: { path: { type: 'string', required: true } },
  839. async execute(args) {
  840. return [{ type: 'text', text: args.path }]
  841. },
  842. }))
  843. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'reader', arguments: {} })
  844. expect(result.isError).toBe(true)
  845. expect(result.content[0]).toMatchObject({
  846. text: 'Error: invalid arguments: missing required property "path"',
  847. })
  848. })
  849. it('runs execute normally when args are valid', async () => {
  850. const ctx = await setup()
  851. ctx.tools.register(defineTool({
  852. name: 'reader',
  853. description: 'reads a path',
  854. parameters: { path: { type: 'string', required: true } },
  855. async execute(args) {
  856. return [{ type: 'text', text: `read ${args.path}` }]
  857. },
  858. }))
  859. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'reader', arguments: { path: '/x' } })
  860. expect(result).toEqual({ callId: CallId('c1'), content: [{ type: 'text', text: 'read /x' }], isError: false })
  861. })
  862. it('ToolArgsError carries a stable code and the violation list', () => {
  863. const err = new ToolArgsError(['missing required property "a"', '"b" must be a number'])
  864. expect(err).toBeInstanceOf(Error)
  865. expect(err.name).toBe('ToolArgsError')
  866. expect(err.code).toBe('INVALID_ARGS')
  867. expect(err.violations).toEqual(['missing required property "a"', '"b" must be a number'])
  868. expect(err.message).toBe('invalid arguments: missing required property "a"; "b" must be a number')
  869. })
  870. it('a schema-invalid call surfaces the structured error on the result', async () => {
  871. const ctx = await setup()
  872. ctx.tools.register(defineTool({
  873. name: 'reader',
  874. description: 'reads a path',
  875. parameters: { path: { type: 'string', required: true } },
  876. async execute(args) {
  877. return [{ type: 'text', text: args.path }]
  878. },
  879. }))
  880. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'reader', arguments: {} })
  881. expect(result.isError).toBe(true)
  882. expect(result.error).toEqual({ name: 'ToolArgsError', code: 'INVALID_ARGS' })
  883. })
  884. it('a tool throwing a HarnessError surfaces its name and code', async () => {
  885. const { HarnessError } = await import('@deepseek-ai/dsh-llm')
  886. const ctx = await setup()
  887. ctx.tools.register({
  888. ...echoTool,
  889. name: 'coded',
  890. async execute() {
  891. throw new HarnessError('disk full', 'ENOSPC')
  892. },
  893. })
  894. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'coded', arguments: {} })
  895. expect(result.isError).toBe(true)
  896. expect(result.error).toEqual({ name: 'HarnessError', code: 'ENOSPC' })
  897. expect(result.content[0]).toMatchObject({ text: 'Error: disk full' })
  898. })
  899. it('a non-HarnessError throw has no structured error (only the text)', async () => {
  900. const ctx = await setup()
  901. ctx.tools.register({
  902. ...echoTool,
  903. name: 'plain',
  904. async execute() {
  905. throw new Error('just a message')
  906. },
  907. })
  908. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'plain', arguments: {} })
  909. expect(result.isError).toBe(true)
  910. expect(result.error).toBeUndefined()
  911. expect(result.content[0]).toMatchObject({ text: 'Error: just a message' })
  912. })
  913. it('raw-registered tools are NOT validated by defineTool (MCP keeps its own)', async () => {
  914. const ctx = await setup()
  915. // A raw ToolDefinition: no defineTool wrapping, so no validateArgs guard.
  916. ctx.tools.register({
  917. name: 'raw',
  918. description: 'raw tool',
  919. parameters: { type: 'object', properties: { path: { type: 'string' } }, required: ['path'] },
  920. async execute(args: unknown) {
  921. return [{ type: 'text', text: typeof args }]
  922. },
  923. })
  924. // Missing the "required" path — but raw tools validate their own input, so
  925. // this reaches execute rather than being rejected by the harness.
  926. const result = await ctx.tools.execute({ callId: CallId('c1'), name: 'raw', arguments: {} })
  927. expect(result.isError).toBe(false)
  928. })
  929. })
  930. describe('defineTool presentation (presentCall / presentResult)', () => {
  931. it('threads presentCall/presentResult onto the ToolDefinition with typed args', () => {
  932. const tool = defineTool({
  933. name: 'demo',
  934. description: 'demo',
  935. parameters: { path: { type: 'string', required: true }, n: { type: 'number' } },
  936. async execute() { return [{ type: 'text', text: 'ok' }] },
  937. presentCall(args) {
  938. // args is typed { path: string; n?: number } — zero casts.
  939. expectTypeOf(args).toEqualTypeOf<{ path: string; n?: number }>()
  940. return { card: 'generic', title: `Open ${args.path}`, kind: 'read', rawInput: args.path }
  941. },
  942. presentResult(args, result) {
  943. return { card: 'generic', title: `Opened ${args.path}`, content: result.content }
  944. },
  945. })
  946. expect(tool.presentCall!({ path: '/a', n: 2 })).toEqual({ card: 'generic', title: 'Open /a', kind: 'read', rawInput: '/a' })
  947. expect(tool.presentResult!({ path: '/a' }, { content: [{ type: 'text', text: 'x' }], isError: false }))
  948. .toEqual({ card: 'generic', title: 'Opened /a', content: [{ type: 'text', text: 'x' }] })
  949. })
  950. it('a tool without presentCall/presentResult leaves them undefined (UI falls back generically)', () => {
  951. const tool = defineTool({
  952. name: 'plain',
  953. description: 'plain',
  954. parameters: { x: { type: 'string', required: true } },
  955. async execute() { return [] },
  956. })
  957. expect(typeof tool.presentCall).toBe('undefined')
  958. expect(typeof tool.presentResult).toBe('undefined')
  959. })
  960. it('presentCall/presentResult validate softly: malformed args return undefined, never throw (display runs on replay)', () => {
  961. const tool = defineTool({
  962. name: 'demo',
  963. description: 'demo',
  964. parameters: { path: { type: 'string', required: true } },
  965. async execute() { return [] },
  966. presentCall: args => ({ card: 'generic', title: args.path }),
  967. presentResult: (args, result) => ({ card: 'generic', title: args.path, content: result.content }),
  968. })
  969. // Unlike execute (which throws ToolArgsError on a mismatch), the display
  970. // methods soft-validate and fall back to undefined so a UI never crashes
  971. // replaying an old/foreign log entry. The ToolDefinition methods take
  972. // `unknown`, so malformed shapes pass without a cast.
  973. expect(tool.presentCall?.({})).toBeUndefined()
  974. expect(tool.presentResult?.({ wrong: 1 }, { content: [], isError: false })).toBeUndefined()
  975. })
  976. })