session-models.host.spec.ts 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663
  1. /**
  2. * Session Controller model-directory and selection behavior: dynamic provider grouping,
  3. * provider-local catalog failures, logged-selection restoration without stale
  4. * catalog injection, advisory pass-through models, and the prompt-assembly
  5. * boundary for a running selection change.
  6. */
  7. import { describe, expect, it, vi } from 'vitest'
  8. import { Context } from '@deepseek-ai/cordis'
  9. import AgentRegistry, { agentEvents } from '@deepseek-ai/dsh-agent'
  10. import type { Agent } from '@deepseek-ai/dsh-agent'
  11. import AttachmentStore from '@deepseek-ai/dsh-attachment'
  12. import LlmRuntime, { LlmAdapter, ReasoningEffortId } from '@deepseek-ai/dsh-llm'
  13. import type {
  14. GenerateOptions, LlmCallConfig, LlmModelInfo, LlmModelReasoningInfo, LlmProviderInfo,
  15. LlmResolvedModelInfo, StreamChunk,
  16. UserMessage,
  17. } from '@deepseek-ai/dsh-llm'
  18. import SessionStore from '@deepseek-ai/dsh-session'
  19. import type { SessionId } from '@deepseek-ai/dsh-session'
  20. import type { SessionPromptRequest, SessionRequestId } from '../src/types.ts'
  21. import SystemPrompt from '@deepseek-ai/dsh-system-prompt'
  22. import { TypertRemoteFailure } from '@deepseek-ai/dsh-typert-protocol'
  23. import { createSessionTestRemote } from './test-remote.ts'
  24. function request<P>(payload: P): P {
  25. return payload
  26. }
  27. let nextRequestId = 1
  28. function promptRequest(
  29. payload: Omit<SessionPromptRequest, 'requestId'>,
  30. ): SessionPromptRequest {
  31. return {
  32. ...payload,
  33. requestId: `models-${String(nextRequestId++)}` as SessionRequestId,
  34. }
  35. }
  36. class CatalogAdapter extends LlmAdapter {
  37. constructor(
  38. private readonly name: string,
  39. private readonly models: readonly LlmModelInfo[] | Error,
  40. private readonly reasoning?: LlmModelReasoningInfo,
  41. private readonly exactError?: Error,
  42. ) {
  43. super()
  44. }
  45. override providerInfo(provider: string): LlmProviderInfo {
  46. return { id: provider, name: this.name }
  47. }
  48. override listModels(): Promise<readonly LlmModelInfo[]> {
  49. return this.models instanceof Error
  50. ? Promise.reject(this.models)
  51. : Promise.resolve(this.models)
  52. }
  53. override resolveModel(provider: string, model: string): Promise<LlmResolvedModelInfo> {
  54. if (this.exactError !== undefined) return Promise.reject(this.exactError)
  55. return Promise.resolve({
  56. provider,
  57. id: model,
  58. name: model,
  59. ...this.reasoning === undefined ? {} : { reasoning: this.reasoning },
  60. })
  61. }
  62. override async *stream(_options: GenerateOptions): AsyncIterable<StreamChunk> {
  63. // Catalog tests never enter provider streaming.
  64. }
  65. }
  66. const REASONING: LlmModelReasoningInfo = {
  67. efforts: [
  68. { id: ReasoningEffortId('off'), name: 'Off' },
  69. { id: ReasoningEffortId('high'), name: 'High' },
  70. { id: ReasoningEffortId('max'), name: 'Max' },
  71. ],
  72. defaultEffort: ReasoningEffortId('high'),
  73. }
  74. async function harness(logged?: {
  75. provider: string
  76. model: string
  77. reasoningEffort?: ReasoningEffortId
  78. }): Promise<{
  79. ctx: Context
  80. agent: Agent
  81. sessionId: SessionId
  82. }> {
  83. const ctx = new Context()
  84. await ctx.plugin(SessionStore)
  85. await ctx.plugin(SystemPrompt, { persona: '' })
  86. await ctx.plugin(LlmRuntime)
  87. await ctx.plugin(AgentRegistry)
  88. ctx.llm.registerAdapter(['deepseek-official'], new CatalogAdapter('DeepSeek', [
  89. { provider: 'deepseek-official', id: 'deepseek-chat', name: 'DeepSeek Chat' },
  90. { provider: 'deepseek-official', id: 'deepseek-reasoner', name: 'DeepSeek Reasoner', description: 'Reasoning model' },
  91. ], REASONING))
  92. ctx.llm.registerAdapter(['broken'], new CatalogAdapter('Broken Provider', new Error('catalog offline')))
  93. ctx.llm.registerAdapter(['metadata-broken'], new CatalogAdapter('Metadata Broken', [
  94. { provider: 'metadata-broken', id: 'listed', name: 'Listed' },
  95. ], undefined, new Error('reasoning metadata offline')))
  96. ctx.llm.registerAdapter(['remote-rejected'], new CatalogAdapter(
  97. 'Remote Rejected',
  98. [],
  99. undefined,
  100. new TypertRemoteFailure({
  101. code: 'fixture-rejected',
  102. message: 'fixture rejected the selection',
  103. details: { provider: 'remote-rejected' },
  104. }),
  105. ))
  106. ctx.llm.registerAdapter(['empty'], new CatalogAdapter('Empty Provider', []))
  107. ctx.llm.registerAdapter(['duplicate'], new CatalogAdapter('Duplicate Provider', [
  108. { provider: 'duplicate', id: 'same', name: 'Same' },
  109. { provider: 'duplicate', id: 'same', name: 'Same Again' },
  110. ]))
  111. const session = ctx.sessions.create()
  112. if (logged !== undefined) {
  113. session.append('request/header', { header: { config: logged }, reason: 'initial' })
  114. }
  115. const agent = {
  116. id: session.id,
  117. session,
  118. status: 'running',
  119. ctx,
  120. inbox: { nextTurn: [], nextStep: [] },
  121. } as unknown as Agent
  122. ctx.agents.register(agent)
  123. return { ctx, agent, sessionId: session.id }
  124. }
  125. function expectValue<T>(result: { ok: true; value: T } | { ok: false }): T {
  126. if (!result.ok) throw new Error('expected successful response')
  127. return result.value
  128. }
  129. function registerTextOnly(ctx: Context): void {
  130. ctx.llm.registerAdapter(['text-only'], new class extends CatalogAdapter {
  131. override resolveModel(provider: string, model: string): Promise<LlmResolvedModelInfo> {
  132. return Promise.resolve({ provider, id: model, name: model, inputModalities: ['text'] })
  133. }
  134. }('Text Only', []))
  135. }
  136. describe('Web session model selection', () => {
  137. it('validates an ordered image batch before persisting any member', async () => {
  138. const { ctx, agent, sessionId } = await harness()
  139. const validateImage = vi.fn((_input: { data: Uint8Array }) => Promise.resolve())
  140. const saveImage = vi.fn((input: { data: Uint8Array; mediaType: 'image/png'; name?: string }) => Promise.resolve({
  141. attachmentId: `att-${String(input.data[0])}`,
  142. mediaType: input.mediaType,
  143. bytes: input.data.byteLength,
  144. width: 1,
  145. height: 1,
  146. ...input.name === undefined ? {} : { name: input.name },
  147. }))
  148. const attachments = {
  149. imageLimits: {
  150. maxImageBytes: 4,
  151. maxImagesPerMessage: 2,
  152. maxMessageImageBytes: 4,
  153. maxImagePixels: 4,
  154. maxImageDimension: 2000,
  155. mediaTypes: ['image/png'],
  156. },
  157. validateImage,
  158. saveImage,
  159. }
  160. ctx.provide('attachments', Object.setPrototypeOf(attachments, AttachmentStore.prototype) as never)
  161. const followup = vi.fn()
  162. Object.assign(agent, { followup })
  163. const remote = createSessionTestRemote(ctx, {
  164. defaultModelSelection: () => ({ provider: 'deepseek-official', model: 'deepseek-chat' }),
  165. cwd: '/tmp',
  166. })
  167. const result = await remote.prompt(promptRequest({
  168. sessionId,
  169. mode: 'queue' as const,
  170. content: [
  171. { type: 'image' as const, mediaType: 'image/png' as const, data: 'AQ==', name: 'first.png' },
  172. { type: 'text' as const, text: 'compare' },
  173. { type: 'image' as const, mediaType: 'image/png' as const, data: 'Ag==' },
  174. ],
  175. }))
  176. expect(result.ok).toBe(true)
  177. expect(validateImage.mock.calls.map(([input]) => [...input.data])).toEqual([[1], [2]])
  178. expect(saveImage.mock.calls.map(([input]) => [...input.data])).toEqual([[1], [2]])
  179. expect((followup.mock.calls[0]?.[0] as UserMessage).content).toEqual([
  180. {
  181. type: 'image',
  182. attachment: {
  183. attachmentId: 'att-1', mediaType: 'image/png', bytes: 1, width: 1, height: 1, name: 'first.png',
  184. },
  185. },
  186. { type: 'text', text: 'compare' },
  187. { type: 'image', attachment: { attachmentId: 'att-2', mediaType: 'image/png', bytes: 1, width: 1, height: 1 } },
  188. ])
  189. const denied = await remote.prompt(promptRequest({
  190. sessionId,
  191. mode: 'queue' as const,
  192. content: Array.from({ length: 3 }, () => ({
  193. type: 'image' as const, mediaType: 'image/png' as const, data: 'AQ==',
  194. })),
  195. }))
  196. expect(denied).toMatchObject({
  197. ok: false,
  198. error: { code: 'attachment-error', details: { reason: 'TOO_MANY_IMAGES' } },
  199. })
  200. expect(saveImage).toHaveBeenCalledTimes(2)
  201. await ctx.fiber.dispose()
  202. })
  203. it('allows a text-only selection while durable or pending images remain available for later models', async () => {
  204. const { ctx, agent, sessionId } = await harness()
  205. registerTextOnly(ctx)
  206. const remote = createSessionTestRemote(ctx, {
  207. defaultModelSelection: () => ({ provider: 'deepseek-official', model: 'deepseek-chat' }),
  208. cwd: '/tmp',
  209. })
  210. const image = {
  211. type: 'image' as const,
  212. attachment: { attachmentId: 'att-history', mediaType: 'image/png' as const, bytes: 1, width: 1, height: 1 },
  213. }
  214. agent.session.append('user/message', {
  215. id: 'image-message', role: 'user', source: { kind: 'user' }, content: [image],
  216. } as never, { surfaceOp: 'append' })
  217. expect(expectValue(await remote.selectModel(request({
  218. sessionId, provider: 'text-only', model: 'plain',
  219. }))).selected).toEqual({ provider: 'text-only', model: 'plain' })
  220. agent.session.append('user/message', {
  221. id: 'summary', role: 'user', source: { kind: 'plugin', plugin: 'compact' },
  222. content: [{ type: 'text', text: 'image summarized' }],
  223. } as never, {
  224. surfaceOp: { op: 'replace', start: 0, end: agent.session.events.length - 1 },
  225. sourceEventSeqs: agent.session.events.map(event => event.seq),
  226. })
  227. ;(agent.inbox.nextTurn as UserMessage[]).push({
  228. id: 'pending-image', role: 'user', source: { kind: 'user' }, content: [image],
  229. } as never)
  230. expect(expectValue(await remote.selectModel(request({
  231. sessionId, provider: 'text-only', model: 'plain',
  232. }))).selected).toEqual({ provider: 'text-only', model: 'plain' })
  233. await ctx.fiber.dispose()
  234. })
  235. it('authorizes attachment bytes only when the session event stream references the id', async () => {
  236. const { ctx, agent, sessionId } = await harness()
  237. const ref = {
  238. attachmentId: 'att-authorized', mediaType: 'image/png' as const, bytes: 2, width: 1, height: 1,
  239. }
  240. const readImage = vi.fn(() => Promise.resolve({ ref, data: Uint8Array.of(1, 2) }))
  241. ctx.provide('attachments', { readImage } as never)
  242. const remote = createSessionTestRemote(ctx, {
  243. defaultModelSelection: () => ({ provider: 'deepseek-official', model: 'deepseek-chat' }),
  244. cwd: '/tmp',
  245. })
  246. agent.session.append('agent/inbox/spliced', {
  247. target: 'next-turn',
  248. start: 0,
  249. inserted: [{
  250. id: 'queued-image', role: 'user', source: { kind: 'user' },
  251. content: [{ type: 'image', attachment: ref }],
  252. }],
  253. } as never)
  254. const allowed = await remote.attachment(request({
  255. sessionId, attachmentId: 'att-authorized' as never,
  256. }))
  257. expect(allowed).toMatchObject({ ok: true, value: { attachment: ref, data: 'AQI=' } })
  258. const denied = await remote.attachment(request({
  259. sessionId, attachmentId: 'att-other' as never,
  260. }))
  261. expect(denied).toMatchObject({
  262. ok: false,
  263. error: { code: 'attachment-error', details: { reason: 'ATTACHMENT_NOT_REFERENCED' } },
  264. })
  265. expect(readImage).toHaveBeenCalledOnce()
  266. await ctx.fiber.dispose()
  267. })
  268. it('groups successful providers and leaves an unlisted current selection out of the catalog', async () => {
  269. const { ctx, sessionId } = await harness({
  270. provider: 'deepseek-official',
  271. model: 'private-preview',
  272. reasoningEffort: ReasoningEffortId('max'),
  273. })
  274. const remote = createSessionTestRemote(ctx, { defaultModelSelection: () => ({ provider: 'deepseek-official', model: 'deepseek-chat' }), cwd: '/tmp' })
  275. const catalog = expectValue(await remote.models(request({ sessionId })))
  276. expect(catalog.current).toEqual({
  277. provider: 'deepseek-official',
  278. model: 'private-preview',
  279. reasoningEffort: 'max',
  280. })
  281. expect(catalog.groups).toEqual([{
  282. id: 'deepseek-official',
  283. name: 'DeepSeek',
  284. models: [
  285. { id: 'deepseek-chat', name: 'DeepSeek Chat', reasoning: REASONING },
  286. {
  287. id: 'deepseek-reasoner',
  288. name: 'DeepSeek Reasoner',
  289. description: 'Reasoning model',
  290. reasoning: REASONING,
  291. },
  292. ],
  293. }])
  294. expect(catalog.failures).toEqual([
  295. { id: 'broken', name: 'Broken Provider', message: 'catalog offline' },
  296. { id: 'metadata-broken', name: 'Metadata Broken', message: 'reasoning metadata offline' },
  297. {
  298. id: 'duplicate',
  299. name: 'Duplicate Provider',
  300. message: 'adapter returned invalid or duplicate model metadata for provider "duplicate"',
  301. },
  302. ])
  303. await ctx.fiber.dispose()
  304. })
  305. it('preserves optional catalog metadata and string provider failures', async () => {
  306. const { ctx, sessionId } = await harness()
  307. ctx.llm.registerAdapter(['plain'], new CatalogAdapter('Plain', [
  308. { provider: 'plain', id: 'plain-model', name: 'Plain Model' },
  309. ]))
  310. ctx.llm.registerAdapter(['described-reasoning'], new CatalogAdapter('Described Reasoning', [
  311. { provider: 'described-reasoning', id: 'reasoning-model', name: 'Reasoning Model' },
  312. ], {
  313. efforts: [{ id: ReasoningEffortId('high'), name: 'High', description: 'More thinking' }],
  314. }))
  315. ctx.llm.registerAdapter(['string-failure'], new class extends CatalogAdapter {
  316. override listModels(): Promise<readonly LlmModelInfo[]> {
  317. // oxlint-disable-next-line typescript/prefer-promise-reject-errors -- non-Error provider normalization is the scenario.
  318. return Promise.reject('string catalog failure')
  319. }
  320. }('String Failure', []))
  321. const remote = createSessionTestRemote(ctx, {
  322. defaultModelSelection: () => ({ provider: 'deepseek-official', model: 'deepseek-chat' }),
  323. cwd: '/tmp',
  324. })
  325. const catalog = expectValue(await remote.models(request({ sessionId })))
  326. expect(catalog.groups).toEqual(expect.arrayContaining([
  327. { id: 'plain', name: 'Plain', models: [{ id: 'plain-model', name: 'Plain Model' }] },
  328. {
  329. id: 'described-reasoning',
  330. name: 'Described Reasoning',
  331. models: [{
  332. id: 'reasoning-model',
  333. name: 'Reasoning Model',
  334. reasoning: {
  335. efforts: [{ id: 'high', name: 'High', description: 'More thinking' }],
  336. },
  337. }],
  338. },
  339. ]))
  340. expect(catalog.failures).toContainEqual({
  341. id: 'string-failure', name: 'String Failure', message: 'string catalog failure',
  342. })
  343. await ctx.fiber.dispose()
  344. })
  345. it('accepts an advisory-unlisted model, rejects an unavailable provider, and switches only after the next assembly', async () => {
  346. const { ctx, agent, sessionId } = await harness()
  347. const remote = createSessionTestRemote(ctx, { defaultModelSelection: () => ({ provider: 'deepseek-official', model: 'deepseek-chat' }), cwd: '/tmp' })
  348. const seed: LlmCallConfig = { provider: 'seed', model: 'seed', temperature: 0.2 }
  349. const signal = new AbortController().signal
  350. expect(expectValue(await remote.models(request({ sessionId }))).current)
  351. .toEqual({ provider: 'deepseek-official', model: 'deepseek-chat' })
  352. expect((await ctx.systemPrompt.assemble()).variables)
  353. .toMatchObject({ provider: 'deepseek-official', model: 'deepseek-chat' })
  354. const selected = expectValue(await remote.selectModel(request({
  355. sessionId,
  356. provider: 'deepseek-official',
  357. model: 'private-preview',
  358. reasoningEffort: 'max',
  359. })))
  360. expect(selected.selected).toEqual({
  361. provider: 'deepseek-official',
  362. model: 'private-preview',
  363. reasoningEffort: 'max',
  364. })
  365. await expect(agentEvents(ctx, agent).waterfall(
  366. 'agent/request', { turn: 1, step: 0, signal }, () => Promise.resolve(seed),
  367. )).resolves.toMatchObject({ provider: 'deepseek-official', model: 'deepseek-chat' })
  368. expect((await ctx.systemPrompt.assemble()).variables)
  369. .toMatchObject({ provider: 'deepseek-official', model: 'private-preview' })
  370. await expect(agentEvents(ctx, agent).waterfall(
  371. 'agent/request', { turn: 1, step: 1, signal }, () => Promise.resolve(seed),
  372. )).resolves.toMatchObject({
  373. provider: 'deepseek-official',
  374. model: 'private-preview',
  375. reasoningEffort: 'max',
  376. })
  377. const unsupported = await remote.selectModel(request({
  378. sessionId,
  379. provider: 'deepseek-official',
  380. model: 'private-preview',
  381. reasoningEffort: 'medium',
  382. }))
  383. expect(unsupported).toMatchObject({
  384. ok: false,
  385. error: {
  386. code: 'model-unavailable',
  387. message: 'provider "deepseek-official" model "private-preview" does not support reasoning effort "medium"',
  388. },
  389. })
  390. const rejected = await remote.selectModel(request({
  391. sessionId,
  392. provider: 'missing',
  393. model: 'model',
  394. }))
  395. expect(rejected).toEqual({
  396. ok: false,
  397. error: {
  398. code: 'model-unavailable',
  399. message: 'no adapter registered for provider "missing"',
  400. details: { provider: 'missing', model: 'model' },
  401. },
  402. })
  403. expect(await remote.selectModel(request({
  404. sessionId,
  405. provider: 'remote-rejected',
  406. model: 'model',
  407. }))).toEqual({
  408. ok: false,
  409. error: {
  410. code: 'fixture-rejected',
  411. message: 'fixture rejected the selection',
  412. details: { provider: 'remote-rejected' },
  413. },
  414. })
  415. expect(expectValue(await remote.models(request({ sessionId }))).current)
  416. .toEqual({ provider: 'deepseek-official', model: 'private-preview', reasoningEffort: 'max' })
  417. await ctx.fiber.dispose()
  418. })
  419. it('reads the Agent default live for a session whose log names no selection', async () => {
  420. const { ctx, sessionId } = await harness()
  421. let stored = { provider: 'deepseek-official', model: 'deepseek-chat' }
  422. const remote = createSessionTestRemote(ctx, {
  423. defaultModelSelection: () => stored,
  424. cwd: '/tmp',
  425. })
  426. expect(expectValue(await remote.models(request({ sessionId }))).current)
  427. .toEqual({ provider: 'deepseek-official', model: 'deepseek-chat' })
  428. // The default moving after the session exists still reaches it: New
  429. // Session reuses a blank session rather than minting another, so a seed
  430. // captured at creation would show the superseded model there.
  431. stored = { provider: 'deepseek-official', model: 'deepseek-reasoner' }
  432. expect(expectValue(await remote.models(request({ sessionId }))).current)
  433. .toEqual({ provider: 'deepseek-official', model: 'deepseek-reasoner' })
  434. await ctx.fiber.dispose()
  435. })
  436. it('keeps a session on its logged selection when the Agent default differs', async () => {
  437. const { ctx, sessionId } = await harness({
  438. provider: 'deepseek-official',
  439. model: 'deepseek-chat',
  440. })
  441. let stored = { provider: 'deepseek-official', model: 'deepseek-chat' }
  442. const remote = createSessionTestRemote(ctx, {
  443. defaultModelSelection: () => stored,
  444. cwd: '/tmp',
  445. })
  446. stored = { provider: 'duplicate', model: 'same' }
  447. expect(expectValue(await remote.models(request({ sessionId }))).current)
  448. .toEqual({ provider: 'deepseek-official', model: 'deepseek-chat' })
  449. await ctx.fiber.dispose()
  450. })
  451. it('saves an accepted selection as the default and survives a storage failure', async () => {
  452. const { ctx, sessionId } = await harness()
  453. const saved: unknown[] = []
  454. let reject = false
  455. const remote = createSessionTestRemote(ctx, {
  456. defaultModelSelection: () => ({ provider: 'deepseek-official', model: 'deepseek-chat' }),
  457. saveDefaultModelSelection: (selection) => {
  458. saved.push(selection)
  459. return reject ? Promise.reject(new Error('read-only document')) : Promise.resolve()
  460. },
  461. cwd: '/tmp',
  462. })
  463. expectValue(await remote.selectModel(request({
  464. sessionId, provider: 'deepseek-official', model: 'deepseek-reasoner', reasoningEffort: 'max',
  465. })))
  466. expect(saved).toEqual([
  467. { provider: 'deepseek-official', model: 'deepseek-reasoner', reasoningEffort: 'max' },
  468. ])
  469. // A refused selection never becomes anyone's default.
  470. await remote.selectModel(request({ sessionId, provider: 'missing', model: 'model' }))
  471. expect(saved).toHaveLength(1)
  472. // Storage failing is not the selection failing: the switch already applies
  473. // to this session, so the call still succeeds.
  474. reject = true
  475. const stillAccepted = expectValue(await remote.selectModel(request({
  476. sessionId, provider: 'deepseek-official', model: 'deepseek-chat',
  477. })))
  478. expect(stillAccepted.selected).toEqual({ provider: 'deepseek-official', model: 'deepseek-chat', reasoningEffort: 'high' })
  479. expect(expectValue(await remote.models(request({ sessionId }))).current)
  480. .toEqual({ provider: 'deepseek-official', model: 'deepseek-chat', reasoningEffort: 'high' })
  481. await ctx.fiber.dispose()
  482. })
  483. it('refuses a prompt no adapter can route, and reports it on the directory', async () => {
  484. const { ctx, sessionId } = await harness()
  485. const remote = createSessionTestRemote(ctx, {
  486. defaultModelSelection: () => ({ provider: 'deleted-gateway', model: 'deleted-model' }),
  487. cwd: '/tmp',
  488. })
  489. // The client disabling its input is an affordance; this method stays
  490. // callable, so the refusal has to live here.
  491. const refused = await remote.prompt(promptRequest({
  492. sessionId, mode: 'queue' as const, content: [{ type: 'text' as const, text: 'hi' }],
  493. }))
  494. expect(refused).toMatchObject({
  495. ok: false,
  496. error: { code: 'model-unavailable', details: { provider: 'deleted-gateway', model: 'deleted-model' } },
  497. })
  498. expect(expectValue(await remote.models(request({ sessionId }))).routable).toBe(false)
  499. // An advisory-unlisted model on a live route is NOT this: the route
  500. // serves it, so the prompt goes through and nothing blocks.
  501. expectValue(await remote.selectModel(request({
  502. sessionId, provider: 'deepseek-official', model: 'unlisted-but-served',
  503. })))
  504. const catalog = expectValue(await remote.models(request({ sessionId })))
  505. expect(catalog.routable).toBe(true)
  506. expect(catalog.groups.flatMap(group => group.models.map(model => model.id)))
  507. .not.toContain('unlisted-but-served')
  508. await ctx.fiber.dispose()
  509. })
  510. it('serves a session and its catalog when the stored default names a route that is gone', async () => {
  511. const { ctx, sessionId } = await harness()
  512. const remote = createSessionTestRemote(ctx, {
  513. // What a Models-page removal leaves behind: the settings document still
  514. // names the route the user last picked, and nothing serves it.
  515. defaultModelSelection: () => ({ provider: 'deleted-gateway', model: 'deleted-model' }),
  516. cwd: '/tmp',
  517. })
  518. const catalog = expectValue(await remote.models(request({ sessionId })))
  519. // Passed through rather than repaired: matching no group is precisely what
  520. // makes the composer seat prompt for a selection instead of naming a model
  521. // the deployment cannot reach.
  522. expect(catalog.current).toEqual({ provider: 'deleted-gateway', model: 'deleted-model' })
  523. expect(catalog.groups.flatMap(group => group.models.map(model => `${group.id}/${model.id}`)))
  524. .not.toContain('deleted-gateway/deleted-model')
  525. await ctx.fiber.dispose()
  526. })
  527. it('maps image admission failures and accepts image-capable selections', async () => {
  528. const { ctx, agent, sessionId } = await harness()
  529. registerTextOnly(ctx)
  530. ctx.llm.registerAdapter(['image-capable'], new class extends CatalogAdapter {
  531. override resolveModel(provider: string, model: string): Promise<LlmResolvedModelInfo> {
  532. return Promise.resolve({
  533. provider, id: model, name: model, inputModalities: ['text', 'image'],
  534. })
  535. }
  536. }('Image Capable', []))
  537. ctx.llm.registerAdapter(['string-error'], new class extends CatalogAdapter {
  538. override resolveModel(): Promise<LlmResolvedModelInfo> {
  539. // oxlint-disable-next-line typescript/prefer-promise-reject-errors -- non-Error provider normalization is the scenario.
  540. return Promise.reject('string selection failure')
  541. }
  542. }('String Error', []))
  543. let saveMode: 'success' | 'error' | 'remote' = 'success'
  544. const savedRef = {
  545. attachmentId: 'saved-image', mediaType: 'image/png' as const, bytes: 1, width: 1, height: 1,
  546. }
  547. ctx.provide('attachments', {
  548. saveImages: () => {
  549. if (saveMode === 'error') return Promise.reject(new Error('image store offline'))
  550. if (saveMode === 'remote') {
  551. return Promise.reject(new TypertRemoteFailure({
  552. code: 'fixture-rejected', message: 'fixture rejected', details: {},
  553. }))
  554. }
  555. return Promise.resolve([savedRef])
  556. },
  557. } as never)
  558. const followup = vi.fn()
  559. Object.assign(agent, { followup })
  560. const remote = createSessionTestRemote(ctx, {
  561. defaultModelSelection: () => ({ provider: 'deepseek-official', model: 'deepseek-chat' }),
  562. cwd: '/tmp',
  563. })
  564. const image = { type: 'image' as const, mediaType: 'image/png' as const, data: 'AQ==' }
  565. expectValue(await remote.selectModel(request({
  566. sessionId, provider: 'text-only', model: 'plain',
  567. })))
  568. expect(await remote.prompt(promptRequest({
  569. sessionId, mode: 'queue', content: [image],
  570. }))).toMatchObject({
  571. ok: false,
  572. error: { code: 'attachment-error', details: { reason: 'MODEL_DOES_NOT_SUPPORT_IMAGES' } },
  573. })
  574. expectValue(await remote.selectModel(request({
  575. sessionId, provider: 'image-capable', model: 'vision',
  576. })))
  577. expect(await remote.prompt(promptRequest({
  578. sessionId, mode: 'queue', content: [{ ...image, data: '' }],
  579. }))).toMatchObject({
  580. ok: false,
  581. error: { code: 'attachment-error', details: { reason: 'INVALID_IMAGE_BASE64' } },
  582. })
  583. saveMode = 'error'
  584. expect(await remote.prompt(promptRequest({
  585. sessionId, mode: 'queue', content: [image],
  586. }))).toMatchObject({ ok: false, error: { code: 'agent-busy' } })
  587. saveMode = 'remote'
  588. expect(await remote.prompt(promptRequest({
  589. sessionId, mode: 'queue', content: [image],
  590. }))).toMatchObject({ ok: false, error: { code: 'fixture-rejected' } })
  591. saveMode = 'success'
  592. expectValue(await remote.prompt(promptRequest({ sessionId, mode: 'queue', content: [image] })))
  593. expect(followup).toHaveBeenCalledOnce()
  594. ;(agent.inbox.nextTurn as UserMessage[]).push({
  595. id: 'pending-image', role: 'user', source: { kind: 'user' },
  596. content: [{ type: 'image', attachment: savedRef }],
  597. } as never)
  598. expectValue(await remote.selectModel(request({
  599. sessionId, provider: 'deepseek-official', model: 'deepseek-chat',
  600. })))
  601. expectValue(await remote.selectModel(request({
  602. sessionId, provider: 'image-capable', model: 'vision',
  603. })))
  604. expect(await remote.selectModel(request({
  605. sessionId, provider: 'metadata-broken', model: 'broken',
  606. }))).toMatchObject({
  607. ok: false, error: { code: 'model-unavailable', message: 'reasoning metadata offline' },
  608. })
  609. expect(await remote.selectModel(request({
  610. sessionId, provider: 'string-error', model: 'broken',
  611. }))).toMatchObject({
  612. ok: false,
  613. error: { code: 'model-unavailable', message: 'string selection failure' },
  614. })
  615. await ctx.fiber.dispose()
  616. })
  617. })