Merge pull request #367 from deepseek-harness/worktree/pi-ai-manual-e2e

ci: add manual pi-ai Azure OpenAI and Anthropic e2e workflow

.github/workflows/pi-ai-provider-e2e.yml

@@ -0,0 +1,78 @@
+name: E2E (pi-ai Azure OpenAI and Anthropic)
+
+# This suite spends tokens against two external providers and is intentionally
+# opt-in. It has no push, pull_request, schedule, or workflow_call trigger.
+on:
+  workflow_dispatch:
+    inputs:
+      azure_openai_model:
+        description: Azure OpenAI model from pi-ai's installed catalog
+        required: true
+        default: gpt-5.5
+        type: string
+      anthropic_model:
+        description: Anthropic model from pi-ai's installed catalog
+        required: true
+        default: claude-opus-4-8
+        type: string
+
+permissions:
+  contents: read
+
+jobs:
+  e2e:
+    runs-on: ubuntu-latest
+    name: Azure OpenAI Responses + Anthropic Messages
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: actions/setup-node@v6
+        with:
+          node-version: 24
+
+      - name: Enable corepack (pnpm)
+        run: corepack enable
+
+      - name: Resolve pnpm store path
+        id: pnpm-store
+        run: echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT"
+
+      - uses: actions/cache@v4
+        with:
+          path: ${{ steps.pnpm-store.outputs.path }}
+          key: ${{ runner.os }}-node-24-pnpm-${{ hashFiles('pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-node-24-pnpm-
+
+      - name: Install (immutable)
+        run: pnpm install --frozen-lockfile
+
+      # The tests self-skip locally when a credential is absent. A manually
+      # dispatched CI run must fail instead of reporting an all-skipped green.
+      - name: Preflight (require provider API keys)
+        env:
+          AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY_EXTERNAL }}
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY_EXTERNAL }}
+        run: |
+          set -euo pipefail
+          missing=0
+          for name in AZURE_OPENAI_API_KEY ANTHROPIC_API_KEY; do
+            if [ -z "${!name:-}" ]; then
+              echo "::error::${name} is empty. Configure the corresponding *_EXTERNAL repository secret."
+              missing=1
+            fi
+          done
+          exit "$missing"
+
+      - name: E2E tests (real Azure OpenAI and Anthropic APIs)
+        env:
+          AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY_EXTERNAL }}
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY_EXTERNAL }}
+          DSH_PI_AI_OPENAI_MODEL: ${{ inputs.azure_openai_model }}
+          DSH_PI_AI_OPENAI_BASE_URL: https://openai-routerhub-resource.services.ai.azure.com/api/projects/openai/openai/v1
+          DSH_PI_AI_ANTHROPIC_MODEL: ${{ inputs.anthropic_model }}
+          DSH_E2E_MAX_WORKERS: 2
+        run: >-
+          pnpm exec vitest run --config vitest.e2e.config.ts
+          packages/llm/llm-pi-ai/tests/provider-apis.e2e.ts

packages/llm/llm-pi-ai/tests/adapter.spec.ts

@@ -168,6 +168,26 @@ describe('PiAiAdapter provider routing', () => {
     expect(server.paths).toEqual(['/v1/responses'])
   })
 
+  it('uses OpenAI Responses against an Azure project v1 path with its API key header', async () => {
+    const server = await mockServer([{ status: 401, body: JSON.stringify({ error: { message: 'expected mock failure' } }) }])
+    const ctx = new Context()
+    await ctx.plugin(LlmService)
+    await ctx.plugin(LlmPiAi, {
+      providers: [{
+        provider: 'openai',
+        apiKey: 'test-key',
+        baseURL: `${server.url}/api/projects/openai/openai/v1`,
+        headers: { 'api-key': 'test-key', Authorization: '' },
+        maxRetries: 0,
+      }],
+    })
+    const result = await assemble(ctx, { provider: 'openai', model: 'gpt-5.5', messages: [] })
+    expect(result.finish.kind).toBe('error')
+    expect(server.paths).toEqual(['/api/projects/openai/openai/v1/responses'])
+    expect(server.headers[0]?.['api-key']).toBe('test-key')
+    expect(server.headers[0]?.authorization).toBe('')
+  })
+
   it.each([
     [401, 'AUTH'],
     [400, 'INVALID_REQUEST'],

packages/llm/llm-pi-ai/tests/provider-apis.e2e.ts

@@ -0,0 +1,163 @@
+import { afterEach, describe, expect, it } from 'vitest'
+import { Context } from 'cordis'
+import LlmService, { CallId } from '@deepseek-ai/dsh-llm'
+import type { Message, ToolSchema } from '@deepseek-ai/dsh-llm'
+import * as LlmPiAi from '@deepseek-ai/dsh-llm-pi-ai'
+import type { PiAiReplayState } from '../src/replay.ts'
+import { assemble, type AssembledResult } from './assemble.ts'
+
+interface ProviderCase {
+  provider: 'openai' | 'anthropic'
+  api: 'openai-responses' | 'anthropic-messages'
+  model: string
+  apiKey?: string
+  baseURL?: string
+  headers?: Record<string, string>
+}
+
+const openAIBaseURL = process.env.DSH_PI_AI_OPENAI_BASE_URL
+const azureOpenAIKey = process.env.AZURE_OPENAI_API_KEY
+
+const providerCases: ProviderCase[] = [
+  {
+    provider: 'openai',
+    api: 'openai-responses',
+    model: process.env.DSH_PI_AI_OPENAI_MODEL ?? 'gpt-5.5',
+    ...azureOpenAIKey
+      ? { apiKey: azureOpenAIKey, headers: { 'api-key': azureOpenAIKey, Authorization: '' } }
+      : {},
+    ...openAIBaseURL ? { baseURL: openAIBaseURL } : {},
+  },
+  {
+    provider: 'anthropic',
+    api: 'anthropic-messages',
+    model: process.env.DSH_PI_AI_ANTHROPIC_MODEL ?? 'claude-opus-4-8',
+    ...process.env.ANTHROPIC_API_KEY ? { apiKey: process.env.ANTHROPIC_API_KEY } : {},
+  },
+]
+
+const contexts: Context[] = []
+
+async function harness(): Promise<Context> {
+  const ctx = new Context()
+  contexts.push(ctx)
+  await ctx.plugin(LlmService)
+  await ctx.plugin(LlmPiAi, {
+    providers: providerCases.map(profile => ({
+      provider: profile.provider,
+      ...profile.apiKey === undefined ? {} : { apiKey: profile.apiKey },
+      ...profile.baseURL === undefined ? {} : { baseURL: profile.baseURL },
+      ...profile.headers === undefined ? {} : { headers: profile.headers },
+    })),
+  })
+  return ctx
+}
+
+afterEach(async () => {
+  await Promise.all(contexts.splice(0).map(ctx => ctx.fiber.dispose()))
+})
+
+function ask(text: string): Message[] {
+  return [{ role: 'user', content: [{ type: 'text', text }] }]
+}
+
+function textOf(result: AssembledResult): string {
+  return result.message.content
+    .filter(block => block.type === 'text')
+    .map(block => block.text)
+    .join('')
+}
+
+function expectFinish(result: AssembledResult, expected: 'stop' | 'tool-calls'): void {
+  if (result.finish.kind === 'error') {
+    throw new Error(`provider request failed (${result.finish.code ?? 'unknown'}): ${result.finish.message}`)
+  }
+  expect(result.finish.kind).toBe(expected)
+}
+
+function expectNativeReplay(result: AssembledResult, profile: ProviderCase): PiAiReplayState {
+  const replayState = result.message.provenance?.replayState
+  expect(replayState).toMatchObject({
+    kind: 'pi-ai',
+    version: 1,
+    api: profile.api,
+    provider: profile.provider,
+    model: profile.model,
+  })
+  return replayState as PiAiReplayState
+}
+
+const lookupTool: ToolSchema = {
+  name: 'lookup_code',
+  description: 'Look up the word represented by a short code.',
+  parameters: {
+    type: 'object',
+    properties: { code: { type: 'string', description: 'The code to look up.' } },
+    required: ['code'],
+  },
+}
+
+for (const profile of providerCases) {
+  describe.skipIf(profile.apiKey === undefined)(
+    `llm-pi-ai ${profile.provider} e2e (${profile.api})`,
+    () => {
+      it('streams text with usage and native replay metadata', async () => {
+        const ctx = await harness()
+        const result = await assemble(ctx, {
+          provider: profile.provider,
+          model: profile.model,
+          messages: ask('Reply with exactly the word: pong'),
+          maxTokens: 1024,
+        })
+
+        expectFinish(result, 'stop')
+        expect(textOf(result).toLowerCase()).toContain('pong')
+        expect(result.usage?.inputTokens).toBeGreaterThan(0)
+        expect(result.usage?.outputTokens).toBeGreaterThan(0)
+        expect(expectNativeReplay(result, profile).stopReason).toBe('stop')
+      })
+
+      it('round-trips a tool call with provider-native replay metadata', async () => {
+        const ctx = await harness()
+        const prompt = ask('Use lookup_code with code "blue". Do not answer without calling the tool.')
+        const first = await assemble(ctx, {
+          provider: profile.provider,
+          model: profile.model,
+          messages: prompt,
+          tools: [lookupTool],
+          maxTokens: 2048,
+        })
+
+        expectFinish(first, 'tool-calls')
+        const call = first.message.content.find(block => block.type === 'tool-call')
+        expect(call).toBeDefined()
+        expect(call!.name).toBe('lookup_code')
+        expect(JSON.parse(call!.arguments)).toMatchObject({ code: 'blue' })
+        expect(expectNativeReplay(first, profile).stopReason).toBe('toolUse')
+
+        const second = await assemble(ctx, {
+          provider: profile.provider,
+          model: profile.model,
+          messages: [
+            ...prompt,
+            first.message,
+            {
+              role: 'user',
+              content: [{
+                type: 'tool-result',
+                toolCallId: CallId(call!.id),
+                content: [{ type: 'text', text: 'The code blue means ocean.' }],
+              }],
+            },
+          ],
+          tools: [lookupTool],
+          maxTokens: 2048,
+        })
+
+        expectFinish(second, 'stop')
+        expect(textOf(second).toLowerCase()).toContain('ocean')
+        expect(expectNativeReplay(second, profile).stopReason).toBe('stop')
+      })
+    },
+  )
+}

vitest.e2e.config.ts

@@ -2,8 +2,9 @@ import tsconfigPaths from 'vite-tsconfig-paths'
 import { defineConfig } from 'vitest/config'
 
 // Real-API suite, separate because it spends tokens. Each test self-skips without
-// DEEPSEEK_API_KEY for keyless CI; the credentialed workflow preflights the secret. Values may come
-// from the environment or gitignored root `.env`, with optional DEEPSEEK_BASE_URL.
+// its provider credential for keyless CI; credentialed workflows preflight the
+// secrets they require. Values may come from the environment or gitignored root
+// `.env`, with provider-specific endpoint overrides where supported.
 try {
   // Node >= 21.7 native; throws when the file does not exist.
   process.loadEnvFile(new URL('.env', import.meta.url).pathname)