read-image.spec.ts 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616
  1. /**
  2. * The `read_image` tool over the REAL local filesystem and attachment store:
  3. * extension routing, the strict image-modality gate (every refusal arm),
  4. * durable commit + image-block rendering, attachment admission failures, and
  5. * the regression that `read` keeps its text-only contract.
  6. */
  7. import { afterEach, beforeEach, describe, expect, it } from 'vitest'
  8. import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
  9. import { tmpdir } from 'node:os'
  10. import { join } from 'node:path'
  11. import { Context } from '@deepseek-ai/cordis'
  12. import { CodeRuntime } from '@deepseek-ai/dsh-code-runtime'
  13. import type { CodeRunRequest, CodeRunResult } from '@deepseek-ai/dsh-code-runtime'
  14. import { CallId, LlmAdapter, LlmRuntime } from '@deepseek-ai/dsh-llm'
  15. import type { GenerateOptions, LlmModelInfo, LlmResolvedModelInfo, Message, StreamChunk } from '@deepseek-ai/dsh-llm'
  16. import SystemPrompt from '@deepseek-ai/dsh-system-prompt'
  17. import ToolRuntime, { RUN_CODE_NAME } from '@deepseek-ai/dsh-tools'
  18. import type { Config as ToolConfig } from '@deepseek-ai/dsh-tools'
  19. import LocalFileSystem from '@deepseek-ai/dsh-fs-local'
  20. import * as FsPolicy from '@deepseek-ai/dsh-fs-observation-policy'
  21. import LocalAttachmentStore from '@deepseek-ai/dsh-attachment-local'
  22. import { AttachmentError, AttachmentId, AttachmentStore } from '@deepseek-ai/dsh-attachment'
  23. import type { ImageAttachmentLimits, ImageAttachmentRef, SaveImageAttachment, StoredImageAttachment } from '@deepseek-ai/dsh-attachment'
  24. import * as ToolFs from '@deepseek-ai/dsh-tool-fs'
  25. import {
  26. applyReadImageTool,
  27. formatImageReadOutput,
  28. imageMediaTypeForPath,
  29. imageRefFromValue,
  30. } from '../src/read-image.ts'
  31. /** 1x1 red PNG (valid signature, IHDR, IDAT). */
  32. const PNG_1X1 = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4z8AAAAMBAQDJ/pLvAAAAAElFTkSuQmCC', 'base64')
  33. /** 3x3 red PNG used to trip a tiny configured pixel limit. */
  34. const PNG_3X3 = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAMAAAADCAIAAADZSiLoAAAAEElEQVR4nGP4z8AAQQxYWACPjgj4kWPEuQAAAABJRU5ErkJggg==', 'base64')
  35. const testToolSignal = new AbortController().signal
  36. /** Exact-route fake adapter; `stream` is unreachable in these tests. */
  37. class CatalogAdapter extends LlmAdapter {
  38. constructor(
  39. private readonly models: LlmModelInfo[],
  40. private readonly resolvedModels: LlmModelInfo[] = models,
  41. ) {
  42. super()
  43. }
  44. override listModels(_provider: string): Promise<readonly LlmModelInfo[]> {
  45. return Promise.resolve(this.models)
  46. }
  47. override resolveModel(provider: string, model: string): Promise<LlmResolvedModelInfo> {
  48. const resolved = this.resolvedModels.find(candidate => candidate.id === model)
  49. return Promise.resolve({
  50. provider,
  51. id: model,
  52. name: resolved?.name ?? model,
  53. ...resolved?.inputModalities === undefined ? {} : { inputModalities: [...resolved.inputModalities] },
  54. })
  55. }
  56. override stream(_options: GenerateOptions): AsyncIterable<StreamChunk> {
  57. throw new Error('read_image tests never stream')
  58. }
  59. }
  60. /** In-process Code Mode seam fake that invokes the real registry bindings. */
  61. class FakeRuntime extends CodeRuntime {
  62. readonly language = 'typescript'
  63. readonly isolation = 'fake'
  64. behavior: (request: CodeRunRequest) => Promise<CodeRunResult> = () => Promise.resolve({ logs: [] })
  65. run(request: CodeRunRequest): Promise<CodeRunResult> {
  66. return this.behavior(request)
  67. }
  68. }
  69. let dir: string
  70. let home: string
  71. beforeEach(async () => {
  72. dir = await mkdtemp(join(tmpdir(), 'dsh-read-image-'))
  73. home = await mkdtemp(join(tmpdir(), 'dsh-read-image-home-'))
  74. })
  75. afterEach(async () => {
  76. await rm(dir, { recursive: true, force: true })
  77. await rm(home, { recursive: true, force: true })
  78. })
  79. interface SetupOptions {
  80. models?: LlmModelInfo[]
  81. resolvedModels?: LlmModelInfo[]
  82. attachments?: boolean
  83. llm?: boolean
  84. storeConfig?: { maxImageBytes?: number; maxImagePixels?: number; maxImageDimension?: number; maxMessageImageBytes?: number }
  85. toolMode?: ToolConfig['mode']
  86. }
  87. async function setup(options: SetupOptions = {}) {
  88. const ctx = new Context()
  89. await ctx.plugin(SystemPrompt)
  90. await ctx.plugin(ToolRuntime, { mode: options.toolMode ?? 'native' })
  91. if (options.toolMode === 'code' || options.toolMode === 'both') {
  92. await ctx.plugin(FakeRuntime)
  93. }
  94. await ctx.plugin(LocalFileSystem, { cwd: dir })
  95. await ctx.plugin(FsPolicy)
  96. if (options.attachments !== false) {
  97. await ctx.plugin(LocalAttachmentStore, { dshHome: home, ...options.storeConfig })
  98. }
  99. if (options.llm !== false) {
  100. await ctx.plugin(LlmRuntime)
  101. ctx.llm.registerAdapter(['visual'], new CatalogAdapter(options.models ?? [
  102. { provider: 'visual', id: 'vision-model', name: 'Vision', inputModalities: ['text', 'image'] },
  103. { provider: 'visual', id: 'text-model', name: 'Text', inputModalities: ['text'] },
  104. { provider: 'visual', id: 'legacy-model', name: 'Legacy' },
  105. ], options.resolvedModels))
  106. }
  107. await ctx.plugin(ToolFs)
  108. return ctx
  109. }
  110. /** A fake calling agent pinned to one routed provider/model. */
  111. function agentOn(model: string | undefined, provider = 'visual', messages: readonly Message[] = []): object {
  112. return {
  113. options: {},
  114. session: {
  115. header: { cwd: dir },
  116. requestHeader: () => (model === undefined ? undefined : { config: { provider, model } }),
  117. deriveMessages: () => [...messages],
  118. append: () => undefined,
  119. },
  120. }
  121. }
  122. let callCounter = 0
  123. function call(ctx: Context, name: string, args: unknown, agent?: object) {
  124. return ctx.tools.execute({
  125. signal: testToolSignal,
  126. callId: CallId(`img-call-${++callCounter}`),
  127. name,
  128. arguments: args,
  129. ...agent ? { agent: agent as never } : {},
  130. })
  131. }
  132. function readImage(ctx: Context, args: unknown, agent?: object) {
  133. return call(ctx, 'read_image', args, agent)
  134. }
  135. function text(result: { content: { type: string; text?: string }[] }): string {
  136. return result.content.filter(b => b.type === 'text').map(b => b.text).join('')
  137. }
  138. describe('imageMediaTypeForPath', () => {
  139. it('maps the four extensions case-insensitively and rejects everything else', () => {
  140. expect(imageMediaTypeForPath('a.png')).toBe('image/png')
  141. expect(imageMediaTypeForPath('a.JPG')).toBe('image/jpeg')
  142. expect(imageMediaTypeForPath('b.jpeg')).toBe('image/jpeg')
  143. expect(imageMediaTypeForPath('c.webp')).toBe('image/webp')
  144. expect(imageMediaTypeForPath('d.Gif')).toBe('image/gif')
  145. expect(imageMediaTypeForPath('note.txt')).toBeUndefined()
  146. expect(imageMediaTypeForPath('png')).toBeUndefined()
  147. })
  148. })
  149. describe('imageRefFromValue', () => {
  150. it('re-brands with and without the optional display name', () => {
  151. const base = { attachmentId: 'sha256:00', mediaType: 'image/png' as const, bytes: 1, width: 1, height: 1 }
  152. expect(imageRefFromValue(base)).toEqual(base)
  153. expect(imageRefFromValue({ ...base, name: 'a.png' })).toEqual({ ...base, name: 'a.png' })
  154. expect(imageRefFromValue({ ...base, originalDimensions: { width: 4, height: 2 } }))
  155. .toEqual({ ...base, originalDimensions: { width: 4, height: 2 } })
  156. })
  157. })
  158. describe('read_image happy path', () => {
  159. it('commits the bytes durably and renders the envelope beside an image block', async () => {
  160. await writeFile(join(dir, 'red.png'), PNG_1X1)
  161. const ctx = await setup()
  162. const result = await readImage(ctx, { file_path: 'red.png' }, agentOn('vision-model'))
  163. expect(result.isError).toBe(false)
  164. expect(result.content).toHaveLength(2)
  165. const image = result.content[1] as { type: string; attachment: ImageAttachmentRef }
  166. expect(image.type).toBe('image')
  167. expect(image.attachment.mediaType).toBe('image/png')
  168. expect(image.attachment.width).toBe(1)
  169. expect(image.attachment.height).toBe(1)
  170. expect(image.attachment.bytes).toBe(PNG_1X1.length)
  171. expect(image.attachment.name).toBe('red.png')
  172. expect(image.attachment.attachmentId).toMatch(/^sha256:[0-9a-f]{64}$/)
  173. expect(text(result)).toBe(formatImageReadOutput(join(dir, 'red.png'), {
  174. attachmentId: image.attachment.attachmentId,
  175. mediaType: 'image/png',
  176. bytes: PNG_1X1.length,
  177. width: 1,
  178. height: 1,
  179. }))
  180. // The committed object must read back verbatim through the store.
  181. const attachments = ctx.get('attachments')
  182. if (attachments === undefined) throw new Error('expected the attachment service')
  183. const stored = await attachments.readImage(image.attachment)
  184. expect(Buffer.from(stored.data)).toEqual(PNG_1X1)
  185. })
  186. it('emits fs/observed for the read image', async () => {
  187. await writeFile(join(dir, 'red.png'), PNG_1X1)
  188. const ctx = await setup()
  189. const observed: string[] = []
  190. ctx.on('fs/observed', target => void observed.push(target.displayPath))
  191. await readImage(ctx, { file_path: 'red.png' }, agentOn('vision-model'))
  192. expect(observed).toEqual([join(dir, 'red.png')])
  193. })
  194. it('falls back to agent options when no request header exists yet', async () => {
  195. await writeFile(join(dir, 'red.png'), PNG_1X1)
  196. const ctx = await setup()
  197. const agent = {
  198. options: { provider: 'visual', model: 'vision-model' },
  199. session: { header: { cwd: dir }, requestHeader: () => undefined },
  200. }
  201. const result = await readImage(ctx, { file_path: 'red.png' }, agent)
  202. expect(result.isError).toBe(false)
  203. })
  204. it('forwards a nested Code Mode image through the outer run_code context', async () => {
  205. await writeFile(join(dir, 'red.png'), PNG_1X1)
  206. const ctx = await setup({ toolMode: 'code' })
  207. const runtime = ctx.codeRuntime as FakeRuntime
  208. runtime.behavior = async (request) => {
  209. const value = await request.bindings[0]!.functions.read_image!({ file_path: 'red.png' })
  210. return { logs: [], value }
  211. }
  212. const result = await call(ctx, RUN_CODE_NAME, {
  213. code: 'return await tools.read_image({ file_path: "red.png" })',
  214. description: 'Read the image through Code Mode',
  215. }, agentOn('vision-model'))
  216. expect(result.isError).toBe(false)
  217. expect(result.content.every(block => block.type === 'text')).toBe(true)
  218. expect(result.additionalContexts).toHaveLength(1)
  219. const forwarded = result.additionalContexts?.[0]?.content
  220. expect(forwarded).toHaveLength(2)
  221. expect(forwarded?.[0]?.type).toBe('text')
  222. expect(forwarded?.[0]?.type === 'text' ? forwarded[0].text : '').toContain('<type>image</type>')
  223. expect(forwarded?.[1]).toMatchObject({
  224. type: 'image',
  225. attachment: { mediaType: 'image/png', width: 1, height: 1 },
  226. })
  227. })
  228. })
  229. describe('strict image-modality gate', () => {
  230. it('accepts an exact visual route even when the advisory model catalog omits it', async () => {
  231. await writeFile(join(dir, 'red.png'), PNG_1X1)
  232. const ctx = await setup({
  233. models: [],
  234. resolvedModels: [
  235. { provider: 'visual', id: 'hidden-vision', name: 'Hidden Vision', inputModalities: ['text', 'image'] },
  236. ],
  237. })
  238. const result = await readImage(ctx, { file_path: 'red.png' }, agentOn('hidden-vision'))
  239. expect(result.isError).toBe(false)
  240. })
  241. it.each([
  242. ['a text-only model', 'text-model'],
  243. ['a model without declared modalities', 'legacy-model'],
  244. ['a model absent from the catalog', 'unknown-model'],
  245. ])('refuses on %s', async (_label, model) => {
  246. await writeFile(join(dir, 'red.png'), PNG_1X1)
  247. const ctx = await setup()
  248. const result = await readImage(ctx, { file_path: 'red.png' }, agentOn(model))
  249. expect(result.isError).toBe(true)
  250. expect(text(result)).toContain('does not declare image input')
  251. })
  252. it('refuses when the route cannot be resolved (no agent, or no header and no options)', async () => {
  253. await writeFile(join(dir, 'red.png'), PNG_1X1)
  254. const ctx = await setup()
  255. const noAgent = await readImage(ctx, { file_path: 'red.png' })
  256. expect(noAgent.isError).toBe(true)
  257. expect(text(noAgent)).toContain('route could not be resolved')
  258. const noRoute = await readImage(ctx, { file_path: 'red.png' }, agentOn(undefined))
  259. expect(noRoute.isError).toBe(true)
  260. expect(text(noRoute)).toContain('route could not be resolved')
  261. })
  262. it('refuses when no llm service is mounted', async () => {
  263. await writeFile(join(dir, 'red.png'), PNG_1X1)
  264. const ctx = await setup({ llm: false })
  265. const result = await readImage(ctx, { file_path: 'red.png' }, agentOn('vision-model'))
  266. expect(result.isError).toBe(true)
  267. expect(text(result)).toContain('route could not be resolved')
  268. })
  269. })
  270. describe('argument and service preconditions', () => {
  271. it('rejects an empty path and a non-image extension', async () => {
  272. const ctx = await setup()
  273. const empty = await readImage(ctx, { file_path: ' ' }, agentOn('vision-model'))
  274. expect(empty.isError).toBe(true)
  275. expect(text(empty)).toContain('non-empty')
  276. const nonImage = await readImage(ctx, { file_path: 'notes.txt' }, agentOn('vision-model'))
  277. expect(nonImage.isError).toBe(true)
  278. expect(text(nonImage)).toContain('only accepts PNG/JPEG/WebP/GIF paths')
  279. })
  280. it('refuses when no attachment service is mounted', async () => {
  281. await writeFile(join(dir, 'red.png'), PNG_1X1)
  282. const ctx = await setup({ attachments: false })
  283. expect(ctx.tools.get('read_image')).toBeUndefined()
  284. expect(ctx.tools.schemas().map(schema => schema.name)).not.toContain('read_image')
  285. const result = await readImage(ctx, { file_path: 'red.png' }, agentOn('vision-model'))
  286. expect(result.isError).toBe(true)
  287. expect(text(result)).toContain('unknown tool "read_image"')
  288. })
  289. it('defensively refuses execution without an attachment service', async () => {
  290. await writeFile(join(dir, 'red.png'), PNG_1X1)
  291. const ctx = await setup({ attachments: false })
  292. applyReadImageTool(ctx)
  293. const result = await readImage(ctx, { file_path: 'red.png' }, agentOn('vision-model'))
  294. expect(result.isError).toBe(true)
  295. expect(text(result)).toContain('no attachment service is mounted')
  296. })
  297. it('refuses a media type the deployment does not accept', async () => {
  298. /** Store whose deployment accepts JPEG only. */
  299. class JpegOnlyStore extends AttachmentStore {
  300. readonly imageLimits: ImageAttachmentLimits = Object.freeze({
  301. maxImageBytes: 1024,
  302. maxImagesPerMessage: 1,
  303. maxMessageImageBytes: 1024,
  304. maxImagePixels: 100,
  305. maxImageDimension: 2000,
  306. mediaTypes: Object.freeze(['image/jpeg'] as const),
  307. })
  308. validateImage(_input: SaveImageAttachment): Promise<void> {
  309. throw new Error('unreachable: admission refuses before validation')
  310. }
  311. saveImage(_input: SaveImageAttachment): Promise<ImageAttachmentRef> {
  312. throw new Error('unreachable: admission refuses before save')
  313. }
  314. readImage(_ref: ImageAttachmentRef): Promise<StoredImageAttachment> {
  315. throw new Error('unreachable in this test')
  316. }
  317. }
  318. const ctx = await setup({ attachments: false })
  319. await ctx.plugin(JpegOnlyStore)
  320. const result = await readImage(ctx, { file_path: 'red.png' }, agentOn('vision-model'))
  321. expect(result.isError).toBe(true)
  322. expect(text(result)).toContain('image/png images are not accepted by this deployment')
  323. })
  324. })
  325. describe('image admission failures', () => {
  326. it('explains how to repair a declared/actual media-type mismatch', async () => {
  327. await writeFile(join(dir, 'wrong.jpg'), PNG_1X1)
  328. const ctx = await setup()
  329. const result = await readImage(ctx, { file_path: 'wrong.jpg' }, agentOn('vision-model'))
  330. expect(result.isError).toBe(true)
  331. expect(text(result)).toContain('the .jpg extension declares image/jpeg')
  332. expect(text(result)).toContain('rename the file to match its actual format if it is PNG/JPEG/WebP/GIF, or convert it to one of those formats')
  333. })
  334. it('fails with FS_TOO_LARGE before reading a file past maxImageBytes', async () => {
  335. await writeFile(join(dir, 'red.png'), PNG_1X1)
  336. const ctx = await setup({ storeConfig: { maxImageBytes: PNG_1X1.length - 1 } })
  337. const result = await readImage(ctx, { file_path: 'red.png' }, agentOn('vision-model'))
  338. expect(result.isError).toBe(true)
  339. expect(text(result)).toContain('exceeds')
  340. })
  341. it('honors the tighter per-message aggregate byte bound', async () => {
  342. await writeFile(join(dir, 'red.png'), PNG_1X1)
  343. const ctx = await setup({ storeConfig: { maxMessageImageBytes: PNG_1X1.length - 1 } })
  344. const result = await readImage(ctx, { file_path: 'red.png' }, agentOn('vision-model'))
  345. expect(result.isError).toBe(true)
  346. expect(text(result)).toContain('exceeds')
  347. })
  348. it('surfaces the pixel limit from the attachment admission', async () => {
  349. await writeFile(join(dir, 'big.png'), PNG_3X3)
  350. const ctx = await setup({ storeConfig: { maxImagePixels: 4 } })
  351. const result = await readImage(ctx, { file_path: 'big.png' }, agentOn('vision-model'))
  352. expect(result.isError).toBe(true)
  353. expect(text(result)).toContain('exceeds the 4-pixel decoded-size limit')
  354. expect(text(result)).toContain('downscale the image and read the smaller copy')
  355. })
  356. it('surfaces the per-side limit from attachment admission', async () => {
  357. await writeFile(join(dir, 'wide.png'), PNG_3X3)
  358. const ctx = await setup({ storeConfig: { maxImageDimension: 2 } })
  359. const result = await readImage(ctx, { file_path: 'wide.png' }, agentOn('vision-model'))
  360. expect(result.isError).toBe(true)
  361. expect(text(result)).toContain('at least one image side exceeds the 2px limit')
  362. expect(text(result)).toContain('downscale the image and read the smaller copy')
  363. })
  364. it('passes storage faults and non-attachment failures through unchanged', async () => {
  365. /** Store whose commit fails with a configurable error; admission itself passes. */
  366. class FailingStore extends AttachmentStore {
  367. static failure: unknown
  368. readonly imageLimits: ImageAttachmentLimits = Object.freeze({
  369. maxImageBytes: 1024,
  370. maxImagesPerMessage: 1,
  371. maxMessageImageBytes: 1024,
  372. maxImagePixels: 100,
  373. maxImageDimension: 2000,
  374. mediaTypes: Object.freeze(['image/png'] as const),
  375. })
  376. validateImage(_input: SaveImageAttachment): Promise<void> {
  377. return Promise.resolve()
  378. }
  379. async saveImage(_input: SaveImageAttachment): Promise<ImageAttachmentRef> {
  380. throw FailingStore.failure
  381. }
  382. readImage(_ref: ImageAttachmentRef): Promise<StoredImageAttachment> {
  383. throw new Error('unreachable in this test')
  384. }
  385. }
  386. await writeFile(join(dir, 'red.png'), PNG_1X1)
  387. const ctx = await setup({ attachments: false })
  388. await ctx.plugin(FailingStore)
  389. FailingStore.failure = new AttachmentError('Unable to persist image attachment.', 'ATTACHMENT_WRITE_FAILED')
  390. const storageFault = await readImage(ctx, { file_path: 'red.png' }, agentOn('vision-model'))
  391. expect(storageFault.isError).toBe(true)
  392. expect(text(storageFault)).toContain('Unable to persist image attachment.')
  393. FailingStore.failure = new AttachmentError(
  394. 'The 16-bit PNG could not be converted to the normalized 8-bit sRGB form.',
  395. 'ATTACHMENT_WRITE_FAILED',
  396. )
  397. const sixteenBit = await readImage(ctx, { file_path: 'red.png' }, agentOn('vision-model'))
  398. expect(text(sixteenBit)).toContain(
  399. `cannot read "${join(dir, 'red.png')}": the 16-bit PNG could not be converted to the normalized 8-bit sRGB form; convert it to an 8-bit PNG/JPEG/WebP and retry`,
  400. )
  401. FailingStore.failure = new AttachmentError('Image cannot be encoded within the configured normalized-image byte cap.', 'IMAGE_TOO_LARGE')
  402. const overBudget = await readImage(ctx, { file_path: 'red.png' }, agentOn('vision-model'))
  403. expect(overBudget.isError).toBe(true)
  404. expect(text(overBudget)).toContain('cannot be stored within the deployment\'s byte limits; downscale the image and read the smaller copy')
  405. FailingStore.failure = new Error('unrelated infrastructure failure')
  406. const unrelated = await readImage(ctx, { file_path: 'red.png' }, agentOn('vision-model'))
  407. expect(unrelated.isError).toBe(true)
  408. expect(text(unrelated)).toContain('unrelated infrastructure failure')
  409. })
  410. it('reports a missing image file and a directory target through the fs vocabulary', async () => {
  411. await mkdir(join(dir, 'folder.png'))
  412. const ctx = await setup()
  413. const observed: { path: string; kind: string }[] = []
  414. ctx.on('fs/observed', (target, observation) => void observed.push({ path: target.displayPath, kind: observation.kind }))
  415. const missing = await readImage(ctx, { file_path: 'absent.png' }, agentOn('vision-model'))
  416. expect(missing.isError).toBe(true)
  417. expect(text(missing)).toContain('not found')
  418. expect(observed).toEqual([{ path: join(dir, 'absent.png'), kind: 'absent' }])
  419. const directory = await readImage(ctx, { file_path: 'folder.png' }, agentOn('vision-model'))
  420. expect(directory.isError).toBe(true)
  421. expect(text(directory)).toContain('not a regular file')
  422. })
  423. it('omits the display name when the store returns a reference without one', async () => {
  424. /** Store echoing a fixed nameless reference; deployments may strip names entirely. */
  425. class NamelessStore extends AttachmentStore {
  426. readonly imageLimits: ImageAttachmentLimits = Object.freeze({
  427. maxImageBytes: 1024,
  428. maxImagesPerMessage: 1,
  429. maxMessageImageBytes: 1024,
  430. maxImagePixels: 100,
  431. maxImageDimension: 2000,
  432. mediaTypes: Object.freeze(['image/png'] as const),
  433. })
  434. validateImage(_input: SaveImageAttachment): Promise<void> {
  435. return Promise.resolve()
  436. }
  437. async saveImage(input: SaveImageAttachment): Promise<ImageAttachmentRef> {
  438. return { attachmentId: AttachmentId('sha256:feed'), mediaType: input.mediaType, bytes: input.data.length, width: 1, height: 1 }
  439. }
  440. readImage(_ref: ImageAttachmentRef): Promise<StoredImageAttachment> {
  441. throw new Error('unreachable in this test')
  442. }
  443. }
  444. await writeFile(join(dir, 'red.png'), PNG_1X1)
  445. const ctx = await setup({ attachments: false })
  446. await ctx.plugin(NamelessStore)
  447. const result = await readImage(ctx, { file_path: 'red.png' }, agentOn('vision-model'))
  448. expect(result.isError).toBe(false)
  449. const image = result.content[1] as { attachment: ImageAttachmentRef }
  450. expect(image.attachment.name).toBeUndefined()
  451. })
  452. it('names the on-disk dimensions and coordinate multiplier when storage downscales', async () => {
  453. /** Store whose normalized image halves the input on both sides. */
  454. class DownscalingStore extends AttachmentStore {
  455. readonly imageLimits: ImageAttachmentLimits = Object.freeze({
  456. maxImageBytes: 1024,
  457. maxImagesPerMessage: 1,
  458. maxMessageImageBytes: 1024,
  459. maxImagePixels: 100,
  460. maxImageDimension: 2000,
  461. mediaTypes: Object.freeze(['image/png'] as const),
  462. })
  463. validateImage(_input: SaveImageAttachment): Promise<void> {
  464. return Promise.resolve()
  465. }
  466. async saveImage(input: SaveImageAttachment): Promise<ImageAttachmentRef> {
  467. return {
  468. attachmentId: AttachmentId('sha256:feed'),
  469. mediaType: input.mediaType,
  470. bytes: 7,
  471. width: 2,
  472. height: 1,
  473. originalDimensions: { width: 4, height: 2 },
  474. }
  475. }
  476. readImage(_ref: ImageAttachmentRef): Promise<StoredImageAttachment> {
  477. throw new Error('unreachable in this test')
  478. }
  479. }
  480. await writeFile(join(dir, 'red.png'), PNG_1X1)
  481. const ctx = await setup({ attachments: false })
  482. await ctx.plugin(DownscalingStore)
  483. const result = await readImage(ctx, { file_path: 'red.png' }, agentOn('vision-model'))
  484. expect(result.isError).toBe(false)
  485. expect(text(result)).toContain('image/png image, 2x1 px, 7 bytes (downscaled from 4x2 px; multiply coordinates by 2.00 to locate features in the original file)')
  486. })
  487. it('names per-axis multipliers when integer rounding makes the ratios differ', () => {
  488. const envelope = formatImageReadOutput('/img/photo.jpg', {
  489. attachmentId: 'sha256:feed', mediaType: 'image/jpeg', bytes: 9, width: 2, height: 1,
  490. originalDimensions: { width: 5, height: 2 },
  491. })
  492. expect(envelope).toContain('downscaled from 5x2 px; multiply x coordinates by 2.50 and y coordinates by 2.00 to locate features in the original file')
  493. })
  494. })
  495. describe('registration surface', () => {
  496. it('withdraws read_image when the tool-fs fiber or the attachment store is disposed (HMR safety)', async () => {
  497. const ctx = new Context()
  498. await ctx.plugin(SystemPrompt)
  499. await ctx.plugin(ToolRuntime, { mode: 'native' })
  500. await ctx.plugin(LocalFileSystem, { cwd: dir })
  501. await ctx.plugin(FsPolicy)
  502. const attachmentsFiber = await ctx.plugin(LocalAttachmentStore, { dshHome: home })
  503. const toolFsFiber = await ctx.plugin(ToolFs)
  504. const names = () => ctx.tools.schemas().map(schema => schema.name).sort()
  505. expect(names()).toEqual(['edit', 'read', 'read_image', 'write'])
  506. // Disposing only the attachment store tears down the scoped inject fiber:
  507. // read_image withdraws while the unconditional tools stay registered.
  508. await attachmentsFiber.dispose()
  509. expect(names()).toEqual(['edit', 'read', 'write'])
  510. // Remounting the store restores the conditional registration.
  511. const remounted = await ctx.plugin(LocalAttachmentStore, { dshHome: home })
  512. expect(names()).toEqual(['edit', 'read', 'read_image', 'write'])
  513. // Disposing the whole plugin withdraws every tool, read_image included.
  514. await toolFsFiber.dispose()
  515. expect(names()).toEqual([])
  516. await remounted.dispose()
  517. })
  518. it('declares read_image parallel-safe and presents a read-family card', async () => {
  519. const ctx = await setup()
  520. expect(ctx.tools.executionMode({
  521. signal: testToolSignal, callId: CallId('img-parallel'), name: 'read_image', arguments: { file_path: 'a.png' },
  522. })).toEqual({ kind: 'parallel' })
  523. expect(ctx.tools.get('read_image')?.presentCall?.({ file_path: 'shot.png' })).toEqual({
  524. card: 'generic',
  525. title: 'Read image shot.png',
  526. kind: 'read',
  527. locations: [{ path: 'shot.png' }],
  528. })
  529. })
  530. })
  531. describe('read keeps its text-only contract', () => {
  532. it('still refuses a PNG as a binary file and line-numbers text', async () => {
  533. await writeFile(join(dir, 'red.png'), PNG_1X1)
  534. await writeFile(join(dir, 'note.txt'), 'hello\nworld')
  535. const ctx = await setup()
  536. const png = await call(ctx, 'read', { file_path: 'red.png' }, agentOn('vision-model'))
  537. expect(png.isError).toBe(true)
  538. expect(text(png)).toContain('binary file')
  539. const txt = await call(ctx, 'read', { file_path: 'note.txt' }, agentOn('text-model'))
  540. expect(txt.isError).toBe(false)
  541. expect(text(txt)).toContain('1: hello')
  542. expect(text(txt)).toContain('<type>file</type>')
  543. })
  544. })