session-models.host.spec.ts 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671
  1. /**
  2. * Session Controller model-directory and selection behavior: dynamic provider grouping,
  3. * provider-local catalog failures, logged-selection restoration without stale
  4. * catalog injection, advisory pass-through models, and the prompt-assembly
  5. * boundary for a running selection change.
  6. */
  7. import { describe, expect, it, vi } from 'vitest'
  8. import { Context } from '@deepseek-ai/cordis'
  9. import AgentRegistry, { agentEvents } from '@deepseek-ai/dsh-agent'
  10. import type { Agent } from '@deepseek-ai/dsh-agent'
  11. import AttachmentStore from '@deepseek-ai/dsh-attachment'
  12. import LlmRuntime, { LlmAdapter, ReasoningEffortId } from '@deepseek-ai/dsh-llm'
  13. import type {
  14. GenerateOptions, LlmCallConfig, LlmModelInfo, LlmModelReasoningInfo, LlmProviderInfo,
  15. LlmResolvedModelInfo, StreamChunk,
  16. UserMessage,
  17. } from '@deepseek-ai/dsh-llm'
  18. import SessionStore from '@deepseek-ai/dsh-session'
  19. import type { SessionId } from '@deepseek-ai/dsh-session'
  20. import type { SessionPromptRequest, SessionRequestId } from '../src/types.ts'
  21. import { buildModelCatalog } from '../src/catalog.ts'
  22. import SystemPrompt from '@deepseek-ai/dsh-system-prompt'
  23. import { TypertRemoteFailure } from '@deepseek-ai/dsh-typert-protocol'
  24. import { createSessionTestRemote } from './test-remote.ts'
  25. function request<P>(payload: P): P {
  26. return payload
  27. }
  28. let nextRequestId = 1
  29. function promptRequest(
  30. payload: Omit<SessionPromptRequest, 'requestId'>,
  31. ): SessionPromptRequest {
  32. return {
  33. ...payload,
  34. requestId: `models-${String(nextRequestId++)}` as SessionRequestId,
  35. }
  36. }
  37. class CatalogAdapter extends LlmAdapter {
  38. constructor(
  39. private readonly name: string,
  40. private readonly models: readonly LlmModelInfo[] | Error,
  41. private readonly reasoning?: LlmModelReasoningInfo,
  42. private readonly exactError?: Error,
  43. ) {
  44. super()
  45. }
  46. override providerInfo(provider: string): LlmProviderInfo {
  47. return { id: provider, name: this.name }
  48. }
  49. override listModels(): Promise<readonly LlmModelInfo[]> {
  50. return this.models instanceof Error
  51. ? Promise.reject(this.models)
  52. : Promise.resolve(this.models)
  53. }
  54. override resolveModel(provider: string, model: string): Promise<LlmResolvedModelInfo> {
  55. if (this.exactError !== undefined) return Promise.reject(this.exactError)
  56. return Promise.resolve({
  57. provider,
  58. id: model,
  59. name: model,
  60. ...this.reasoning === undefined ? {} : { reasoning: this.reasoning },
  61. })
  62. }
  63. override async *stream(_options: GenerateOptions): AsyncIterable<StreamChunk> {
  64. // Catalog tests never enter provider streaming.
  65. }
  66. }
  67. const REASONING: LlmModelReasoningInfo = {
  68. efforts: [
  69. { id: ReasoningEffortId('off'), name: 'Off' },
  70. { id: ReasoningEffortId('high'), name: 'High' },
  71. { id: ReasoningEffortId('max'), name: 'Max' },
  72. ],
  73. defaultEffort: ReasoningEffortId('high'),
  74. }
  75. async function harness(logged?: {
  76. provider: string
  77. model: string
  78. reasoningEffort?: ReasoningEffortId
  79. }): Promise<{
  80. ctx: Context
  81. agent: Agent
  82. sessionId: SessionId
  83. }> {
  84. const ctx = new Context()
  85. await ctx.plugin(SessionStore)
  86. await ctx.plugin(SystemPrompt, { persona: '' })
  87. await ctx.plugin(LlmRuntime)
  88. await ctx.plugin(AgentRegistry)
  89. ctx.llm.registerAdapter(['deepseek-official'], new CatalogAdapter('DeepSeek', [
  90. { provider: 'deepseek-official', id: 'deepseek-chat', name: 'DeepSeek Chat' },
  91. { provider: 'deepseek-official', id: 'deepseek-reasoner', name: 'DeepSeek Reasoner', description: 'Reasoning model' },
  92. ], REASONING))
  93. ctx.llm.registerAdapter(['broken'], new CatalogAdapter('Broken Provider', new Error('catalog offline')))
  94. ctx.llm.registerAdapter(['metadata-broken'], new CatalogAdapter('Metadata Broken', [
  95. { provider: 'metadata-broken', id: 'listed', name: 'Listed' },
  96. ], undefined, new Error('reasoning metadata offline')))
  97. ctx.llm.registerAdapter(['remote-rejected'], new CatalogAdapter(
  98. 'Remote Rejected',
  99. [],
  100. undefined,
  101. new TypertRemoteFailure({
  102. code: 'fixture-rejected',
  103. message: 'fixture rejected the selection',
  104. details: { provider: 'remote-rejected' },
  105. }),
  106. ))
  107. ctx.llm.registerAdapter(['empty'], new CatalogAdapter('Empty Provider', []))
  108. ctx.llm.registerAdapter(['duplicate'], new CatalogAdapter('Duplicate Provider', [
  109. { provider: 'duplicate', id: 'same', name: 'Same' },
  110. { provider: 'duplicate', id: 'same', name: 'Same Again' },
  111. ]))
  112. const session = ctx.sessions.create()
  113. if (logged !== undefined) {
  114. session.append('request/header', { header: { config: logged }, reason: 'initial' })
  115. }
  116. const agent = {
  117. id: session.id,
  118. session,
  119. status: 'running',
  120. ctx,
  121. inbox: { nextTurn: [], nextStep: [] },
  122. } as unknown as Agent
  123. ctx.agents.register(agent)
  124. return { ctx, agent, sessionId: session.id }
  125. }
  126. function expectValue<T>(result: { ok: true; value: T } | { ok: false }): T {
  127. if (!result.ok) throw new Error('expected successful response')
  128. return result.value
  129. }
  130. function registerTextOnly(ctx: Context): void {
  131. ctx.llm.registerAdapter(['text-only'], new class extends CatalogAdapter {
  132. override resolveModel(provider: string, model: string): Promise<LlmResolvedModelInfo> {
  133. return Promise.resolve({ provider, id: model, name: model, inputModalities: ['text'] })
  134. }
  135. }('Text Only', []))
  136. }
  137. /** Resolve the Client-visible next selection from durable state and the Host default. */
  138. function currentSelection(ctx: Context, sessionId: SessionId) {
  139. const session = ctx.sessions.get(sessionId)
  140. if (session === undefined) throw new Error('expected a live test Session')
  141. return ctx.sessionProjections.snapshot(session).values.modelSelection?.next
  142. ?? ctx.agentDefaultModel.currentSelection()
  143. }
  144. describe('Web session model selection', () => {
  145. it('validates an ordered image batch before persisting any member', async () => {
  146. const { ctx, agent, sessionId } = await harness()
  147. const validateImage = vi.fn((_input: { data: Uint8Array }) => Promise.resolve())
  148. const saveImage = vi.fn((input: { data: Uint8Array; mediaType: 'image/png'; name?: string }) => Promise.resolve({
  149. attachmentId: `att-${String(input.data[0])}`,
  150. mediaType: input.mediaType,
  151. bytes: input.data.byteLength,
  152. width: 1,
  153. height: 1,
  154. ...input.name === undefined ? {} : { name: input.name },
  155. }))
  156. const attachments = {
  157. imageLimits: {
  158. maxImageBytes: 4,
  159. maxImagesPerMessage: 2,
  160. maxMessageImageBytes: 4,
  161. maxImagePixels: 4,
  162. maxImageDimension: 2000,
  163. mediaTypes: ['image/png'],
  164. },
  165. validateImage,
  166. saveImage,
  167. }
  168. ctx.provide('attachments', Object.setPrototypeOf(attachments, AttachmentStore.prototype) as never)
  169. const followup = vi.fn()
  170. Object.assign(agent, { followup })
  171. const remote = createSessionTestRemote(ctx, {
  172. defaultModelSelection: () => ({ provider: 'deepseek-official', model: 'deepseek-chat' }),
  173. cwd: '/tmp',
  174. })
  175. const result = await remote.prompt(promptRequest({
  176. sessionId,
  177. mode: 'queue' as const,
  178. content: [
  179. { type: 'image' as const, mediaType: 'image/png' as const, data: 'AQ==', name: 'first.png' },
  180. { type: 'text' as const, text: 'compare' },
  181. { type: 'image' as const, mediaType: 'image/png' as const, data: 'Ag==' },
  182. ],
  183. }))
  184. expect(result.ok).toBe(true)
  185. expect(validateImage.mock.calls.map(([input]) => [...input.data])).toEqual([[1], [2]])
  186. expect(saveImage.mock.calls.map(([input]) => [...input.data])).toEqual([[1], [2]])
  187. expect((followup.mock.calls[0]?.[0] as UserMessage).content).toEqual([
  188. {
  189. type: 'image',
  190. attachment: {
  191. attachmentId: 'att-1', mediaType: 'image/png', bytes: 1, width: 1, height: 1, name: 'first.png',
  192. },
  193. },
  194. { type: 'text', text: 'compare' },
  195. { type: 'image', attachment: { attachmentId: 'att-2', mediaType: 'image/png', bytes: 1, width: 1, height: 1 } },
  196. ])
  197. const denied = await remote.prompt(promptRequest({
  198. sessionId,
  199. mode: 'queue' as const,
  200. content: Array.from({ length: 3 }, () => ({
  201. type: 'image' as const, mediaType: 'image/png' as const, data: 'AQ==',
  202. })),
  203. }))
  204. expect(denied).toMatchObject({
  205. ok: false,
  206. error: { code: 'attachment-error', details: { reason: 'TOO_MANY_IMAGES' } },
  207. })
  208. expect(saveImage).toHaveBeenCalledTimes(2)
  209. await ctx.fiber.dispose()
  210. })
  211. it('allows a text-only selection while durable or pending images remain available for later models', async () => {
  212. const { ctx, agent, sessionId } = await harness()
  213. registerTextOnly(ctx)
  214. const remote = createSessionTestRemote(ctx, {
  215. defaultModelSelection: () => ({ provider: 'deepseek-official', model: 'deepseek-chat' }),
  216. cwd: '/tmp',
  217. })
  218. const image = {
  219. type: 'image' as const,
  220. attachment: { attachmentId: 'att-history', mediaType: 'image/png' as const, bytes: 1, width: 1, height: 1 },
  221. }
  222. const imageEvent = agent.session.append('user/message', {
  223. id: 'image-message', role: 'user', source: { kind: 'user' }, content: [image],
  224. } as never, { surfaceOp: 'append' })
  225. expect(expectValue(await remote.selectModel(request({
  226. sessionId, provider: 'text-only', model: 'plain',
  227. }))).selected).toEqual({ provider: 'text-only', model: 'plain' })
  228. agent.session.append('user/message', {
  229. id: 'summary', role: 'user', source: { kind: 'plugin', plugin: 'compact' },
  230. content: [{ type: 'text', text: 'image summarized' }],
  231. } as never, {
  232. surfaceOp: { op: 'replace', start: imageEvent.seq, end: imageEvent.seq },
  233. sourceEventSeqs: [imageEvent.seq],
  234. })
  235. ;(agent.inbox.nextTurn as UserMessage[]).push({
  236. id: 'pending-image', role: 'user', source: { kind: 'user' }, content: [image],
  237. } as never)
  238. expect(expectValue(await remote.selectModel(request({
  239. sessionId, provider: 'text-only', model: 'plain',
  240. }))).selected).toEqual({ provider: 'text-only', model: 'plain' })
  241. await ctx.fiber.dispose()
  242. })
  243. it('authorizes attachment bytes only when the session event stream references the id', async () => {
  244. const { ctx, agent, sessionId } = await harness()
  245. const ref = {
  246. attachmentId: 'att-authorized', mediaType: 'image/png' as const, bytes: 2, width: 1, height: 1,
  247. }
  248. const readImage = vi.fn(() => Promise.resolve({ ref, data: Uint8Array.of(1, 2) }))
  249. ctx.provide('attachments', { readImage } as never)
  250. const remote = createSessionTestRemote(ctx, {
  251. defaultModelSelection: () => ({ provider: 'deepseek-official', model: 'deepseek-chat' }),
  252. cwd: '/tmp',
  253. })
  254. agent.session.append('agent/inbox/spliced', {
  255. target: 'next-turn',
  256. start: 0,
  257. inserted: [{
  258. id: 'queued-image', role: 'user', source: { kind: 'user' },
  259. content: [{ type: 'image', attachment: ref }],
  260. }],
  261. } as never)
  262. const allowed = await remote.attachment(request({
  263. sessionId, attachmentId: 'att-authorized' as never,
  264. }))
  265. expect(allowed).toMatchObject({ ok: true, value: { attachment: ref, data: 'AQI=' } })
  266. const denied = await remote.attachment(request({
  267. sessionId, attachmentId: 'att-other' as never,
  268. }))
  269. expect(denied).toMatchObject({
  270. ok: false,
  271. error: { code: 'attachment-error', details: { reason: 'ATTACHMENT_NOT_REFERENCED' } },
  272. })
  273. expect(readImage).toHaveBeenCalledOnce()
  274. await ctx.fiber.dispose()
  275. })
  276. it('groups successful providers and leaves an unlisted current selection out of the catalog', async () => {
  277. const { ctx, sessionId } = await harness({
  278. provider: 'deepseek-official',
  279. model: 'private-preview',
  280. reasoningEffort: ReasoningEffortId('max'),
  281. })
  282. createSessionTestRemote(ctx, { defaultModelSelection: () => ({ provider: 'deepseek-official', model: 'deepseek-chat' }), cwd: '/tmp' })
  283. const catalog = await buildModelCatalog(ctx)
  284. expect(currentSelection(ctx, sessionId)).toEqual({
  285. provider: 'deepseek-official',
  286. model: 'private-preview',
  287. reasoningEffort: 'max',
  288. })
  289. expect(catalog.groups).toEqual([{
  290. id: 'deepseek-official',
  291. name: 'DeepSeek',
  292. models: [
  293. { id: 'deepseek-chat', name: 'DeepSeek Chat', reasoning: REASONING },
  294. {
  295. id: 'deepseek-reasoner',
  296. name: 'DeepSeek Reasoner',
  297. description: 'Reasoning model',
  298. reasoning: REASONING,
  299. },
  300. ],
  301. }])
  302. expect(catalog.failures).toEqual([
  303. { id: 'broken', name: 'Broken Provider', message: 'catalog offline' },
  304. { id: 'metadata-broken', name: 'Metadata Broken', message: 'reasoning metadata offline' },
  305. {
  306. id: 'duplicate',
  307. name: 'Duplicate Provider',
  308. message: 'adapter returned invalid or duplicate model metadata for provider "duplicate"',
  309. },
  310. ])
  311. await ctx.fiber.dispose()
  312. })
  313. it('preserves optional catalog metadata and string provider failures', async () => {
  314. const { ctx } = await harness()
  315. ctx.llm.registerAdapter(['plain'], new CatalogAdapter('Plain', [
  316. { provider: 'plain', id: 'plain-model', name: 'Plain Model' },
  317. ]))
  318. ctx.llm.registerAdapter(['described-reasoning'], new CatalogAdapter('Described Reasoning', [
  319. { provider: 'described-reasoning', id: 'reasoning-model', name: 'Reasoning Model' },
  320. ], {
  321. efforts: [{ id: ReasoningEffortId('high'), name: 'High', description: 'More thinking' }],
  322. }))
  323. ctx.llm.registerAdapter(['string-failure'], new class extends CatalogAdapter {
  324. override listModels(): Promise<readonly LlmModelInfo[]> {
  325. // oxlint-disable-next-line typescript/prefer-promise-reject-errors -- non-Error provider normalization is the scenario.
  326. return Promise.reject('string catalog failure')
  327. }
  328. }('String Failure', []))
  329. createSessionTestRemote(ctx, {
  330. defaultModelSelection: () => ({ provider: 'deepseek-official', model: 'deepseek-chat' }),
  331. cwd: '/tmp',
  332. })
  333. const catalog = await buildModelCatalog(ctx)
  334. expect(catalog.groups).toEqual(expect.arrayContaining([
  335. { id: 'plain', name: 'Plain', models: [{ id: 'plain-model', name: 'Plain Model' }] },
  336. {
  337. id: 'described-reasoning',
  338. name: 'Described Reasoning',
  339. models: [{
  340. id: 'reasoning-model',
  341. name: 'Reasoning Model',
  342. reasoning: {
  343. efforts: [{ id: 'high', name: 'High', description: 'More thinking' }],
  344. },
  345. }],
  346. },
  347. ]))
  348. expect(catalog.failures).toContainEqual({
  349. id: 'string-failure', name: 'String Failure', message: 'string catalog failure',
  350. })
  351. await ctx.fiber.dispose()
  352. })
  353. it('accepts an advisory-unlisted model, rejects an unavailable provider, and switches only after the next assembly', async () => {
  354. const { ctx, agent, sessionId } = await harness()
  355. const remote = createSessionTestRemote(ctx, { defaultModelSelection: () => ({ provider: 'deepseek-official', model: 'deepseek-chat' }), cwd: '/tmp' })
  356. const seed: LlmCallConfig = { provider: 'seed', model: 'seed', temperature: 0.2 }
  357. const signal = new AbortController().signal
  358. expect(currentSelection(ctx, sessionId))
  359. .toEqual({ provider: 'deepseek-official', model: 'deepseek-chat' })
  360. const selected = expectValue(await remote.selectModel(request({
  361. sessionId,
  362. provider: 'deepseek-official',
  363. model: 'private-preview',
  364. reasoningEffort: 'max',
  365. })))
  366. expect(selected.selected).toEqual({
  367. provider: 'deepseek-official',
  368. model: 'private-preview',
  369. reasoningEffort: 'max',
  370. })
  371. await expect(agentEvents(ctx, agent).waterfall(
  372. 'agent/request', { turn: 1, step: 0, signal }, () => Promise.resolve(seed),
  373. )).resolves.toEqual(seed)
  374. expect((await ctx.systemPrompt.assemble()).variables)
  375. .toMatchObject({ provider: 'deepseek-official', model: 'private-preview' })
  376. await expect(agentEvents(ctx, agent).waterfall(
  377. 'agent/request', { turn: 1, step: 1, signal }, () => Promise.resolve(seed),
  378. )).resolves.toMatchObject({
  379. provider: 'deepseek-official',
  380. model: 'private-preview',
  381. reasoningEffort: 'max',
  382. })
  383. const unsupported = await remote.selectModel(request({
  384. sessionId,
  385. provider: 'deepseek-official',
  386. model: 'private-preview',
  387. reasoningEffort: 'medium',
  388. }))
  389. expect(unsupported).toMatchObject({
  390. ok: false,
  391. error: {
  392. code: 'model-unavailable',
  393. message: 'provider "deepseek-official" model "private-preview" does not support reasoning effort "medium"',
  394. },
  395. })
  396. const rejected = await remote.selectModel(request({
  397. sessionId,
  398. provider: 'missing',
  399. model: 'model',
  400. }))
  401. expect(rejected).toEqual({
  402. ok: false,
  403. error: {
  404. code: 'model-unavailable',
  405. message: 'no adapter registered for provider "missing"',
  406. details: { provider: 'missing', model: 'model' },
  407. },
  408. })
  409. expect(await remote.selectModel(request({
  410. sessionId,
  411. provider: 'remote-rejected',
  412. model: 'model',
  413. }))).toEqual({
  414. ok: false,
  415. error: {
  416. code: 'fixture-rejected',
  417. message: 'fixture rejected the selection',
  418. details: { provider: 'remote-rejected' },
  419. },
  420. })
  421. expect(currentSelection(ctx, sessionId))
  422. .toEqual({ provider: 'deepseek-official', model: 'private-preview', reasoningEffort: 'max' })
  423. await ctx.fiber.dispose()
  424. })
  425. it('reads the Agent default live for a session whose log names no selection', async () => {
  426. const { ctx, sessionId } = await harness()
  427. let stored = { provider: 'deepseek-official', model: 'deepseek-chat' }
  428. createSessionTestRemote(ctx, {
  429. defaultModelSelection: () => stored,
  430. cwd: '/tmp',
  431. })
  432. expect(currentSelection(ctx, sessionId))
  433. .toEqual({ provider: 'deepseek-official', model: 'deepseek-chat' })
  434. // The default moving after the session exists still reaches it: New
  435. // Session reuses a blank session rather than minting another, so a seed
  436. // captured at creation would show the superseded model there.
  437. stored = { provider: 'deepseek-official', model: 'deepseek-reasoner' }
  438. expect(currentSelection(ctx, sessionId))
  439. .toEqual({ provider: 'deepseek-official', model: 'deepseek-reasoner' })
  440. await ctx.fiber.dispose()
  441. })
  442. it('keeps a session on its logged selection when the Agent default differs', async () => {
  443. const { ctx, sessionId } = await harness({
  444. provider: 'deepseek-official',
  445. model: 'deepseek-chat',
  446. })
  447. let stored = { provider: 'deepseek-official', model: 'deepseek-chat' }
  448. createSessionTestRemote(ctx, {
  449. defaultModelSelection: () => stored,
  450. cwd: '/tmp',
  451. })
  452. stored = { provider: 'duplicate', model: 'same' }
  453. expect(currentSelection(ctx, sessionId))
  454. .toEqual({ provider: 'deepseek-official', model: 'deepseek-chat' })
  455. await ctx.fiber.dispose()
  456. })
  457. it('saves an accepted selection as the default and survives a storage failure', async () => {
  458. const { ctx, sessionId } = await harness()
  459. const saved: unknown[] = []
  460. let reject = false
  461. const remote = createSessionTestRemote(ctx, {
  462. defaultModelSelection: () => ({ provider: 'deepseek-official', model: 'deepseek-chat' }),
  463. saveDefaultModelSelection: (selection) => {
  464. saved.push(selection)
  465. return reject ? Promise.reject(new Error('read-only document')) : Promise.resolve()
  466. },
  467. cwd: '/tmp',
  468. })
  469. expectValue(await remote.selectModel(request({
  470. sessionId, provider: 'deepseek-official', model: 'deepseek-reasoner', reasoningEffort: 'max',
  471. })))
  472. expect(saved).toEqual([
  473. { provider: 'deepseek-official', model: 'deepseek-reasoner', reasoningEffort: 'max' },
  474. ])
  475. // A refused selection never becomes anyone's default.
  476. await remote.selectModel(request({ sessionId, provider: 'missing', model: 'model' }))
  477. expect(saved).toHaveLength(1)
  478. // Storage failing is not the selection failing: the switch already applies
  479. // to this session, so the call still succeeds.
  480. reject = true
  481. const stillAccepted = expectValue(await remote.selectModel(request({
  482. sessionId, provider: 'deepseek-official', model: 'deepseek-chat',
  483. })))
  484. expect(stillAccepted.selected).toEqual({ provider: 'deepseek-official', model: 'deepseek-chat', reasoningEffort: 'high' })
  485. expect(currentSelection(ctx, sessionId))
  486. .toEqual({ provider: 'deepseek-official', model: 'deepseek-chat', reasoningEffort: 'high' })
  487. await ctx.fiber.dispose()
  488. })
  489. it('refuses a prompt no adapter can route, and reports it on the directory', async () => {
  490. const { ctx, sessionId } = await harness()
  491. const remote = createSessionTestRemote(ctx, {
  492. defaultModelSelection: () => ({ provider: 'deleted-gateway', model: 'deleted-model' }),
  493. cwd: '/tmp',
  494. })
  495. // The client disabling its input is an affordance; this method stays
  496. // callable, so the refusal has to live here.
  497. const refused = await remote.prompt(promptRequest({
  498. sessionId, mode: 'queue' as const, content: [{ type: 'text' as const, text: 'hi' }],
  499. }))
  500. expect(refused).toMatchObject({
  501. ok: false,
  502. error: { code: 'model-unavailable', details: { provider: 'deleted-gateway', model: 'deleted-model' } },
  503. })
  504. const unavailableCatalog = await buildModelCatalog(ctx)
  505. expect(unavailableCatalog.routableProviders.includes(currentSelection(ctx, sessionId).provider)).toBe(false)
  506. // An advisory-unlisted model on a live route is NOT this: the route
  507. // serves it, so the prompt goes through and nothing blocks.
  508. expectValue(await remote.selectModel(request({
  509. sessionId, provider: 'deepseek-official', model: 'unlisted-but-served',
  510. })))
  511. const catalog = await buildModelCatalog(ctx)
  512. expect(catalog.routableProviders.includes(currentSelection(ctx, sessionId).provider)).toBe(true)
  513. expect(catalog.groups.flatMap(group => group.models.map(model => model.id)))
  514. .not.toContain('unlisted-but-served')
  515. await ctx.fiber.dispose()
  516. })
  517. it('serves a session and its catalog when the stored default names a route that is gone', async () => {
  518. const { ctx, sessionId } = await harness()
  519. createSessionTestRemote(ctx, {
  520. // What a Models-page removal leaves behind: the settings document still
  521. // names the route the user last picked, and nothing serves it.
  522. defaultModelSelection: () => ({ provider: 'deleted-gateway', model: 'deleted-model' }),
  523. cwd: '/tmp',
  524. })
  525. const catalog = await buildModelCatalog(ctx)
  526. // Passed through rather than repaired: matching no group is precisely what
  527. // makes the composer seat prompt for a selection instead of naming a model
  528. // the deployment cannot reach.
  529. expect(currentSelection(ctx, sessionId)).toEqual({ provider: 'deleted-gateway', model: 'deleted-model' })
  530. expect(catalog.groups.flatMap(group => group.models.map(model => `${group.id}/${model.id}`)))
  531. .not.toContain('deleted-gateway/deleted-model')
  532. await ctx.fiber.dispose()
  533. })
  534. it('maps image admission failures and accepts image-capable selections', async () => {
  535. const { ctx, agent, sessionId } = await harness()
  536. registerTextOnly(ctx)
  537. ctx.llm.registerAdapter(['image-capable'], new class extends CatalogAdapter {
  538. override resolveModel(provider: string, model: string): Promise<LlmResolvedModelInfo> {
  539. return Promise.resolve({
  540. provider, id: model, name: model, inputModalities: ['text', 'image'],
  541. })
  542. }
  543. }('Image Capable', []))
  544. ctx.llm.registerAdapter(['string-error'], new class extends CatalogAdapter {
  545. override resolveModel(): Promise<LlmResolvedModelInfo> {
  546. // oxlint-disable-next-line typescript/prefer-promise-reject-errors -- non-Error provider normalization is the scenario.
  547. return Promise.reject('string selection failure')
  548. }
  549. }('String Error', []))
  550. let saveMode: 'success' | 'error' | 'remote' = 'success'
  551. const savedRef = {
  552. attachmentId: 'saved-image', mediaType: 'image/png' as const, bytes: 1, width: 1, height: 1,
  553. }
  554. ctx.provide('attachments', {
  555. saveImages: () => {
  556. if (saveMode === 'error') return Promise.reject(new Error('image store offline'))
  557. if (saveMode === 'remote') {
  558. return Promise.reject(new TypertRemoteFailure({
  559. code: 'fixture-rejected', message: 'fixture rejected', details: {},
  560. }))
  561. }
  562. return Promise.resolve([savedRef])
  563. },
  564. } as never)
  565. const followup = vi.fn()
  566. Object.assign(agent, { followup })
  567. const remote = createSessionTestRemote(ctx, {
  568. defaultModelSelection: () => ({ provider: 'deepseek-official', model: 'deepseek-chat' }),
  569. cwd: '/tmp',
  570. })
  571. const image = { type: 'image' as const, mediaType: 'image/png' as const, data: 'AQ==' }
  572. expectValue(await remote.selectModel(request({
  573. sessionId, provider: 'text-only', model: 'plain',
  574. })))
  575. expect(await remote.prompt(promptRequest({
  576. sessionId, mode: 'queue', content: [image],
  577. }))).toMatchObject({
  578. ok: false,
  579. error: { code: 'attachment-error', details: { reason: 'MODEL_DOES_NOT_SUPPORT_IMAGES' } },
  580. })
  581. expectValue(await remote.selectModel(request({
  582. sessionId, provider: 'image-capable', model: 'vision',
  583. })))
  584. expect(await remote.prompt(promptRequest({
  585. sessionId, mode: 'queue', content: [{ ...image, data: '' }],
  586. }))).toMatchObject({
  587. ok: false,
  588. error: { code: 'attachment-error', details: { reason: 'INVALID_IMAGE_BASE64' } },
  589. })
  590. saveMode = 'error'
  591. expect(await remote.prompt(promptRequest({
  592. sessionId, mode: 'queue', content: [image],
  593. }))).toMatchObject({ ok: false, error: { code: 'agent-busy' } })
  594. saveMode = 'remote'
  595. expect(await remote.prompt(promptRequest({
  596. sessionId, mode: 'queue', content: [image],
  597. }))).toMatchObject({ ok: false, error: { code: 'fixture-rejected' } })
  598. saveMode = 'success'
  599. expectValue(await remote.prompt(promptRequest({ sessionId, mode: 'queue', content: [image] })))
  600. expect(followup).toHaveBeenCalledOnce()
  601. ;(agent.inbox.nextTurn as UserMessage[]).push({
  602. id: 'pending-image', role: 'user', source: { kind: 'user' },
  603. content: [{ type: 'image', attachment: savedRef }],
  604. } as never)
  605. expectValue(await remote.selectModel(request({
  606. sessionId, provider: 'deepseek-official', model: 'deepseek-chat',
  607. })))
  608. expectValue(await remote.selectModel(request({
  609. sessionId, provider: 'image-capable', model: 'vision',
  610. })))
  611. expect(await remote.selectModel(request({
  612. sessionId, provider: 'metadata-broken', model: 'broken',
  613. }))).toMatchObject({
  614. ok: false, error: { code: 'model-unavailable', message: 'reasoning metadata offline' },
  615. })
  616. expect(await remote.selectModel(request({
  617. sessionId, provider: 'string-error', model: 'broken',
  618. }))).toMatchObject({
  619. ok: false,
  620. error: { code: 'model-unavailable', message: 'string selection failure' },
  621. })
  622. await ctx.fiber.dispose()
  623. })
  624. })