Browse Source

test(llm): replay Anthropic's documented model-listing example

The List Models reference reply joins the recorded listings so the
anthropic-messages path is pinned to the published response fields.
Yichen Jiang 1 month ago
parent
commit
8de7518c89

+ 2 - 2
.agents/notes/implemented/architecture/2026-09-02-protocol-specific-model-listing-discovery.i18n.yaml

@@ -2,5 +2,5 @@
 # side as of the last confirmed-consistent state. Both languages carry equal authority;
 # after editing either side, bring the other along and re-record with:
 #   pnpm run verify-translation-pairing --write .agents/notes/implemented/architecture/2026-09-02-protocol-specific-model-listing-discovery.md
-2026-09-02-protocol-specific-model-listing-discovery.md: 9e14f0ca82f74c2e428c97cb4d6303a1415c9fec
-2026-09-02-protocol-specific-model-listing-discovery.zh.md: 28c43cb5f6e35198a007cb820a460aa862965c93
+2026-09-02-protocol-specific-model-listing-discovery.md: 69e684c284ff57da7ed61fd5dc417fd110d3e5b7
+2026-09-02-protocol-specific-model-listing-discovery.zh.md: 682806fb72c2cc24572e6e351d4d388cead84dc0

+ 1 - 1
.agents/notes/implemented/architecture/2026-09-02-protocol-specific-model-listing-discovery.md

@@ -40,4 +40,4 @@ The supported formats remain an explicit compatibility set rather than arbitrary
 
 ## Testing
 
-Local HTTP-server tests pin both accepted response formats, field normalization, name fallback, ignored malformed entries, Anthropic headers, the maximum-page query, and both spellings of the Anthropic root. Replies recorded from OpenRouter, models.dev, and DeepSeek on 2026-09-02 live under `packages/llm/llm-pi-ai/tests/fixtures/model-listings/` and replay through the parser, so the accepted field spellings are pinned to real endpoints rather than to hand-written samples.
+Local HTTP-server tests pin both accepted response formats, field normalization, name fallback, ignored malformed entries, Anthropic headers, the maximum-page query, and both spellings of the Anthropic root. Replies recorded from OpenRouter, models.dev, and DeepSeek on 2026-09-02, together with the example reply in Anthropic's List Models reference, live under `packages/llm/llm-pi-ai/tests/fixtures/model-listings/` and replay through the parser, so the accepted field spellings are pinned to real endpoints and the published reference rather than to hand-written samples.

+ 1 - 1
.agents/notes/implemented/architecture/2026-09-02-protocol-specific-model-listing-discovery.zh.md

@@ -40,4 +40,4 @@ Models 页面无需改变请求身份,即可询问 OpenAI 兼容网关与 Anth
 
 ## 测试
 
-本地 HTTP 服务器测试钉住两种受支持响应格式、字段归一化、名称回退、忽略畸形条目、Anthropic 标头、最大页查询,以及 Anthropic 根地址的两种写法。2026-09-02 从 OpenRouter、models.dev 与 DeepSeek 录得的回复存放在 `packages/llm/llm-pi-ai/tests/fixtures/model-listings/` 下并经解析器回放,因此受支持的字段拼写钉在真实端点上,而不是手写样例上。
+本地 HTTP 服务器测试钉住两种受支持响应格式、字段归一化、名称回退、忽略畸形条目、Anthropic 标头、最大页查询,以及 Anthropic 根地址的两种写法。2026-09-02 从 OpenRouter、models.dev 与 DeepSeek 录得的回复,连同 Anthropic List Models 参考文档给出的示例回复,存放在 `packages/llm/llm-pi-ai/tests/fixtures/model-listings/` 下并经解析器回放,因此受支持的字段拼写钉在真实端点与公开参考文档上,而不是手写样例上。

+ 16 - 5
packages/llm/llm-pi-ai/tests/discovery.spec.ts

@@ -485,14 +485,16 @@ describe('probe key format', () => {
 })
 
 /**
- * Replies recorded from live endpoints on 2026-09-02. Each file keeps the
- * reply's top-level fields and entry objects verbatim; only the entry list is
+ * Replies recorded from live endpoints on 2026-09-02, plus the reply
+ * Anthropic's List Models reference documents. Each file keeps the reply's
+ * top-level fields and entry objects verbatim; only a recorded entry list is
  * cut down to the named entries so the archive stays small.
  */
 const RECORDED_LISTINGS = [
   {
     name: 'OpenRouter GET /api/v1/models',
     file: 'openrouter-2026-09-02.json',
+    api: 'openai-completions',
     models: [
       { id: 'anthropic/claude-fable-5.1', name: 'Anthropic: Claude Fable 5.1', contextWindow: 1_000_000, maxTokens: 128_000 },
       // The router's own aggregate route reports no completion cap.
@@ -503,6 +505,7 @@ const RECORDED_LISTINGS = [
   {
     name: 'the models.dev anthropic provider object',
     file: 'models-dev-anthropic-2026-09-02.json',
+    api: 'openai-completions',
     models: [
       { id: 'claude-opus-4-7', name: 'Claude Opus 4.7', contextWindow: 1_000_000, maxTokens: 128_000 },
       { id: 'claude-fable-5-1', name: 'Claude Fable 5.1', contextWindow: 1_000_000, maxTokens: 128_000 },
@@ -512,21 +515,29 @@ const RECORDED_LISTINGS = [
   {
     name: 'DeepSeek GET /models',
     file: 'deepseek-2026-09-02.json',
+    api: 'openai-completions',
     models: [
       { id: 'deepseek-v4-flash', name: 'deepseek-v4-flash' },
       { id: 'deepseek-v4-pro', name: 'deepseek-v4-pro' },
       { id: 'deepseek-v4-flash-vision-exp', name: 'deepseek-v4-flash-vision-exp' },
     ],
   },
+  {
+    name: "Anthropic's documented GET /v1/models example",
+    file: 'anthropic-reference-example.json',
+    api: 'anthropic-messages',
+    // The reference example fills both capacities with 0, which is not a
+    // usable capacity, so the row carries the name alone.
+    models: [{ id: 'claude-opus-5', name: 'Claude Opus 5' }],
+  },
 ]
 
 describe('recorded provider listings', () => {
-  it.each(RECORDED_LISTINGS)('reads $name as recorded', async ({ file, models }) => {
+  it.each(RECORDED_LISTINGS)('reads $name as recorded', async ({ file, api, models }) => {
     const body = await readFile(new URL(`./fixtures/model-listings/${file}`, import.meta.url), 'utf8')
     const server = await listingServer({ body })
     const ctx = await harness()
 
-    await expect(ctx.llm.discoverModels('llm-pi-ai', { baseURL: server.url, api: 'openai-completions' }))
-      .resolves.toEqual(models)
+    await expect(ctx.llm.discoverModels('llm-pi-ai', { baseURL: server.url, api })).resolves.toEqual(models)
   })
 })

+ 76 - 0
packages/llm/llm-pi-ai/tests/fixtures/model-listings/anthropic-reference-example.json

@@ -0,0 +1,76 @@
+{
+  "data": [
+    {
+      "id": "claude-opus-5",
+      "capabilities": {
+        "batch": {
+          "supported": true
+        },
+        "citations": {
+          "supported": true
+        },
+        "code_execution": {
+          "supported": true
+        },
+        "context_management": {
+          "clear_thinking_20251015": {
+            "supported": true
+          },
+          "clear_tool_uses_20250919": {
+            "supported": true
+          },
+          "compact_20260112": {
+            "supported": true
+          },
+          "supported": true
+        },
+        "effort": {
+          "high": {
+            "supported": true
+          },
+          "low": {
+            "supported": true
+          },
+          "max": {
+            "supported": true
+          },
+          "medium": {
+            "supported": true
+          },
+          "supported": true,
+          "xhigh": {
+            "supported": true
+          }
+        },
+        "image_input": {
+          "supported": true
+        },
+        "pdf_input": {
+          "supported": true
+        },
+        "structured_outputs": {
+          "supported": true
+        },
+        "thinking": {
+          "supported": true,
+          "types": {
+            "adaptive": {
+              "supported": true
+            },
+            "enabled": {
+              "supported": true
+            }
+          }
+        }
+      },
+      "created_at": "2026-07-24T00:00:00Z",
+      "display_name": "Claude Opus 5",
+      "max_input_tokens": 0,
+      "max_tokens": 0,
+      "type": "model"
+    }
+  ],
+  "first_id": "first_id",
+  "has_more": true,
+  "last_id": "last_id"
+}